{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 673,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "% reset -f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from numpy.random import seed\n",
    "seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARTICLE_NAME</th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yoplait Berry       Delights 6pk</td>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Heat &amp; Eat          Green Curry Small</td>\n",
       "      <td>SALADS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Emer Lds Tall Boot  Harper 6 11 Tan PP</td>\n",
       "      <td>FAMILY FOOTWEAR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Beef Topside Cap On Ctn</td>\n",
       "      <td>FRESH BEEF SUPPLIES CTN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Natures Way Mini Multi 50 + 50s</td>\n",
       "      <td>VITAMINS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             ARTICLE_NAME              SUBCAT_NAME\n",
       "0        Yoplait Berry       Delights 6pk          DAIRY - YOGHURT\n",
       "1   Heat & Eat          Green Curry Small                   SALADS\n",
       "2  Emer Lds Tall Boot  Harper 6 11 Tan PP          FAMILY FOOTWEAR\n",
       "3                 Beef Topside Cap On Ctn  FRESH BEEF SUPPLIES CTN\n",
       "4         Natures Way Mini Multi 50 + 50s                 VITAMINS"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "import gensim\n",
    "import pandas as pd\n",
    "\n",
    "data=pd.read_table(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/articles.txt\",encoding='latin1')\n",
    "keep_subcats=pd.read_csv(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/keep_subcats.csv\",encoding='latin1')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 9, 6, 4, 0, 3, 1, 7, 8, 5])"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "seed(1)\n",
    "np.random.choice(10,10,replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# data2=data\n",
    "# data2.head()\n",
    "# data2=data.loc[np.random.choice(data.shape[0],data.shape[0],replace=False)]\n",
    "# data2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# data2=data.reset_index()\n",
    "# data2=data2.drop(['index'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# data3=data2\n",
    "# data3['ARTICLE_NAME']=(data['ARTICLE_NAME']+\" \"+data2['ARTICLE_NAME'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# data3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# data=data.append(data3)\n",
    "# data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(360029, 3)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.merge(data,keep_subcats,on=\"SUBCAT_NAME\",how='left')\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARTICLE_NAME</th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "      <th>KEEP_SUBCAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yoplait Berry       Delights 6pk</td>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Heat &amp; Eat          Green Curry Small</td>\n",
       "      <td>SALADS</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Emer Lds Tall Boot  Harper 6 11 Tan PP</td>\n",
       "      <td>FAMILY FOOTWEAR</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Beef Topside Cap On Ctn</td>\n",
       "      <td>FRESH BEEF SUPPLIES CTN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Natures Way Mini Multi 50 + 50s</td>\n",
       "      <td>VITAMINS</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             ARTICLE_NAME              SUBCAT_NAME  \\\n",
       "0        Yoplait Berry       Delights 6pk          DAIRY - YOGHURT   \n",
       "1   Heat & Eat          Green Curry Small                   SALADS   \n",
       "2  Emer Lds Tall Boot  Harper 6 11 Tan PP          FAMILY FOOTWEAR   \n",
       "3                 Beef Topside Cap On Ctn  FRESH BEEF SUPPLIES CTN   \n",
       "4         Natures Way Mini Multi 50 + 50s                 VITAMINS   \n",
       "\n",
       "   KEEP_SUBCAT  \n",
       "0          1.0  \n",
       "1          NaN  \n",
       "2          NaN  \n",
       "3          NaN  \n",
       "4          NaN  "
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARTICLE_NAME</th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "      <th>KEEP_SUBCAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yoplait Berry       Delights 6pk</td>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Heat &amp; Eat          Green Curry Small</td>\n",
       "      <td>unknown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Emer Lds Tall Boot  Harper 6 11 Tan PP</td>\n",
       "      <td>unknown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Beef Topside Cap On Ctn</td>\n",
       "      <td>unknown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Natures Way Mini Multi 50 + 50s</td>\n",
       "      <td>unknown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             ARTICLE_NAME      SUBCAT_NAME  KEEP_SUBCAT\n",
       "0        Yoplait Berry       Delights 6pk  DAIRY - YOGHURT          1.0\n",
       "1   Heat & Eat          Green Curry Small          unknown          NaN\n",
       "2  Emer Lds Tall Boot  Harper 6 11 Tan PP          unknown          NaN\n",
       "3                 Beef Topside Cap On Ctn          unknown          NaN\n",
       "4         Natures Way Mini Multi 50 + 50s          unknown          NaN"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "data['SUBCAT_NAME']=np.where(data['KEEP_SUBCAT']==1,data['SUBCAT_NAME'],\"unknown\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(83570, 3)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_subcats=data[data['SUBCAT_NAME']!=\"unknown\"]\n",
    "data_subcats.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(276459, 3)"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_unknown=data[data['SUBCAT_NAME']==\"unknown\"]\n",
    "data_unknown.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARTICLE_NAME</th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "      <th>KEEP_SUBCAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DVD Teenage Mutant  Ninja Turtles</td>\n",
       "      <td>unknown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Taste of Asia Sweet Chilli Sauce 700ml</td>\n",
       "      <td>unknown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Smitten - 12 Stories</td>\n",
       "      <td>unknown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FRESH AS OREGANO    POWDER 6G</td>\n",
       "      <td>unknown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bk Japanese Phrs 7  Japan 7RRP  14.99</td>\n",
       "      <td>unknown</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             ARTICLE_NAME SUBCAT_NAME  KEEP_SUBCAT\n",
       "0       DVD Teenage Mutant  Ninja Turtles     unknown          NaN\n",
       "1  Taste of Asia Sweet Chilli Sauce 700ml     unknown          NaN\n",
       "2                    Smitten - 12 Stories     unknown          NaN\n",
       "3           FRESH AS OREGANO    POWDER 6G     unknown          NaN\n",
       "4   Bk Japanese Phrs 7  Japan 7RRP  14.99     unknown          NaN"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_unknown=data_unknown.reset_index()\n",
    "data_unknown=data_unknown.drop(['index'],axis=1)\n",
    "random_array=np.random.choice(data_unknown.shape[0],100000,replace=False)\n",
    "data_unknown2=data_unknown.loc[random_array]\n",
    "data_unknown2.head()\n",
    "data_unknown2=data_unknown2.reset_index()\n",
    "data_unknown2=data_unknown2.drop(['index'],axis=1)\n",
    "data_unknown2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(183570, 3)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=data_subcats.append(data_unknown2)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#%reset -f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(183570, 2)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=data.drop([\"KEEP_SUBCAT\"],axis=1)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data=data.reset_index()\n",
    "data=data.drop(['index'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from inflection import singularize\n",
    "def preprocess(text):\n",
    "    text=text.lower()\n",
    "    text=re.sub(\"[^a-zA-Z]+\",\" \",text)\n",
    "    text2=text.split()\n",
    "    #filtered_words = [word for word in text2 if word not in stopwords.words('english')]\n",
    "    filtered_words2 = [word for word in text2 if word not in ['kg','ml','ea','pk','cm','gm','or','the','in','and','is','of','i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', 'couldn', 'didn', 'doesn', 'hadn', 'hasn', 'haven', 'isn', 'ma', 'mightn', 'mustn', 'needn', 'shan', 'shouldn', 'wasn', 'weren', 'won','stock','buy','products','store','shelf','get','brand','always','go','one','time','shop','range','like','brands','limited','low','times','day','free','never','sale','shopping','last','food','variety','choice','back','two','home','good','small','run','found','well','use','something','still','first','number','full','days','less','selection','next','little','months','select','home','brand','special','price','produce','quality','regular','per','size','stocking','better','long','pm','hard','made','make','new','night','morning','way','late','reduced','space','great','display','three','normal','stuff','non','bare','may','place','away','dont','saturday','ever','happy','us','top','feel','take','finding','filled','wish','friday','miss','floor','nice','row','done','start','litter','pick','quick','right','open','reach','fill','online','note','yes','spots','walk','room','cleaning','system','easy','husband','ripe','truck','cases','tags','serve','centre','th','dates','grown','labels','company','key','hour','wife','simple','waste','selections','cabinet','deal','call','fine','inside','along','crushed','felt','lost','wonder','rare','grab','budget','added','trays','six','rush','push','oh','collect','primary','refill','general','previous','superior','always','homebrand','range','cosmetics','finish','dinner','lucky','tourist','season','part','variety','city','happy','answer','previous','see','se','really','bad','shop','complete','max','whole','short','check','out','spot','basic','frozen','business','essential','shelves','child','ad','brief','favourite','pass','note','direct','woolworths','bc','ones','delete','popular','overseas','sourced','mum','often','items','everyday','son','page','break','dozen','half','australian','aussie','mixed','show','set','wide','st','fresh','much','lines','box','end','self','big','nothing','large','local','work','stick','flavours','flavour','date','bit','paper','high','isles','aisle','simply','fresh','best','travel','standards','standard','christmas','foods','old','remote','drive','live','1st','wide','families','extra','four','pack','cut','label','ww','park','house','ready','single','road','flowers','favourite','favourites','australia','fast','hill','love','previous','yrs','sock','wash','training','mths','esp','healthy','gripe','thigh','middle']]\n",
    "    text3=' '.join([singularize(word) for word in filtered_words2 if len(word) >1])\n",
    "    text4=text3.split()\n",
    "    filtered_words3 = [word for word in text4 if word not in ['kg','ml','ea','pk','cm','gm','or','the','in','and','is','of','i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', 'couldn', 'didn', 'doesn', 'hadn', 'hasn', 'haven', 'isn', 'ma', 'mightn', 'mustn', 'needn', 'shan', 'shouldn', 'wasn', 'weren', 'won','stock','buy','products','store','shelf','get','brand','always','go','one','time','shop','range','like','brands','limited','low','times','day','free','never','sale','shopping','last','food','variety','choice','back','two','home','good','small','run','found','well','use','something','still','first','number','full','days','less','selection','next','little','months','select','home','brand','special','price','produce','quality','regular','per','size','stocking','better','long','pm','hard','made','make','new','night','morning','way','late','reduced','space','great','display','three','normal','stuff','non','bare','may','place','away','dont','saturday','ever','happy','us','top','feel','take','finding','filled','wish','friday','miss','floor','nice','row','done','start','litter','pick','quick','right','open','reach','fill','online','note','yes','spots','walk','room','cleaning','system','easy','husband','ripe','truck','cases','tags','serve','centre','th','dates','grown','labels','company','key','hour','wife','simple','waste','selections','cabinet','deal','call','fine','inside','along','crushed','felt','lost','wonder','rare','grab','budget','added','trays','six','rush','push','oh','collect','primary','refill','general','previous','superior','always','homebrand','range','cosmetics','finish','dinner','lucky','tourist','season','part','variety','city','happy','answer','previous','see','se','really','bad','shop','complete','max','whole','short','check','out','spot','basic','frozen','business','essential','shelves','child','ad','brief','favourite','pass','note','direct','woolworths','bc','ones','delete','popular','overseas','sourced','mum','often','items','everyday','son','page','break','dozen','half','australian','aussie','mixed','show','set','wide','st','fresh','much','lines','box','end','self','big','nothing','large','local','work','stick','flavours','flavour','date','bit','paper','high','isles','aisle','simply','fresh','best','travel','standards','standard','christmas','foods','old','remote','drive','live','1st','wide','families','extra','four','pack','cut','label','ww','park','house','ready','single','road','flowers','favourite','favourites','australia','fast','hill','love','previous','yrs','sock','wash','training','mths','esp','healthy','gripe','thigh','middle']]\n",
    "    text5=' '.join([singularize(word) for word in filtered_words3 if len(word) >1])\n",
    "    return(text5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(183570, 2)"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "comments=pd.read_table(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/comments.csv\",encoding='latin1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def comments_preprocess(text):\n",
    "    text=text.lower()\n",
    "    text=re.sub(\"[^a-zA-Z]+\",\" \",text)\n",
    "    text2=text.split()\n",
    "    #filtered_words = [word for word in text2 if word not in stopwords.words('english')]\n",
    "    filtered_words2 = [word for word in text2 if word not in ['kg','ml','ea','pk','cm','gm','or','the','in','and','is','of','i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', 'couldn', 'didn', 'doesn', 'hadn', 'hasn', 'haven', 'isn', 'ma', 'mightn', 'mustn', 'needn', 'shan', 'shouldn', 'wasn', 'weren', 'won','stock','buy','products','store','shelf','get','brand','always','go','one','time','shop','range','like','brands','limited','low','times','day','free','never','sale','shopping','last','food','variety','choice','back','two','home','good','small','run','found','well','use','something','still','first','number','full','days','less','selection','next','little','months','select','home','brand','primary','refill','general','previous','superior','always','homebrand','range','cosmetics','finish','dinner','lucky','tourist','season','part','variety','city','happy','answer','previous','see','se','really','bad','shop','complete','max','whole','short','check','out','spot','basic','frozen','business','essential','shelves','child','ad','brief','favourite','pass','note','direct','woolworths','bc']]\n",
    "    text3=' '.join([word for word in filtered_words2 if len(word) >1])\n",
    "    text4=text3.split()\n",
    "    filtered_words3 = [word for word in text4 if word not in ['kg','ml','ea','pk','cm','gm','or','the','in','and','is','of','i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', 'couldn', 'didn', 'doesn', 'hadn', 'hasn', 'haven', 'isn', 'ma', 'mightn', 'mustn', 'needn', 'shan', 'shouldn', 'wasn', 'weren', 'won','stock','buy','products','store','shelf','get','brand','always','go','one','time','shop','range','like','brands','limited','low','times','day','free','never','sale','shopping','last','food','variety','choice','back','two','home','good','small','run','found','well','use','something','still','first','number','full','days','less','selection','next','little','months','select','home','brand','special','price','produce','quality','regular','per','size','stocking','better','long','pm','hard','made','make','new','night','morning','way','late','reduced','space','great','display','three','normal','stuff','non','bare','may','place','away','dont','saturday','ever','happy','us','top','feel','take','finding','filled','wish','friday','miss','floor','nice','row','done','start','litter','pick','quick','right','open','reach','fill','online','note','yes','spots','walk','room','cleaning','system','easy','husband','ripe','truck','cases','tags','serve','centre','th','dates','grown','labels','company','key','hour','wife','simple','waste','selections','cabinet','deal','call','fine','inside','along','crushed','felt','lost','wonder','rare','grab','budget','added','trays','six','rush','push','oh','collect','primary','refill','general','previous','superior','always','homebrand','range','cosmetics','finish','dinner','lucky','tourist','season','part','variety','city','happy','answer','previous','see','se','really','bad','shop','complete','max','whole','short','check','out','spot','basic','frozen','business','essential','shelves','child','ad','brief','favourite','pass','note','direct','woolworths','bc','ones','delete','popular','overseas','sourced','mum','often','items','everyday','son','page','break','dozen','half','australian','aussie','mixed','show','set','wide','st','fresh','much','lines','box','end','self','big','nothing','large','local','work','stick','flavours','flavour','date','bit','paper','high','isles','aisle','simply','fresh','best','travel','standards','standard','christmas','foods','old','remote','drive','live','1st','wide','families','extra','four','pack','cut','label','ww','park','house','ready','single','road','flowers','favourite','favourites','australia','fast','hill','love']]\n",
    "    text5=' '.join([singularize(word) for word in filtered_words3 if len(word) >1])\n",
    "    return(text5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Comment'], dtype='object')"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>Freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>available</td>\n",
       "      <td>13497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>empty</td>\n",
       "      <td>6314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>wanted</td>\n",
       "      <td>5765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bread</td>\n",
       "      <td>4782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>milk</td>\n",
       "      <td>4717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        word   Freq\n",
       "0  available  13497\n",
       "1      empty   6314\n",
       "2     wanted   5765\n",
       "3      bread   4782\n",
       "4       milk   4717"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments['Comment']=comments['Comment'].apply(comments_preprocess)\n",
    "count_comments=comments['Comment'].str.split(' ', expand=True).stack().value_counts()\n",
    "count_comments2=pd.DataFrame(count_comments)\n",
    "count_comments2=count_comments2.reset_index()\n",
    "count_comments2.columns=[\"word\",\"Freq\"]\n",
    "count_comments2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count_comments2.to_csv(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/comment_word_counts8.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data['ARTICLE_NAME']=data['ARTICLE_NAME'].apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARTICLE_NAME</th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yoplait berry delight</td>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>destiny playstatpre order</td>\n",
       "      <td>PET NEEDS - DOG TREATS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>disney plush</td>\n",
       "      <td>BABY - BASICS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gold alula newborn</td>\n",
       "      <td>BABY FORMULA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rowy caramel fudge</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                ARTICLE_NAME                     SUBCAT_NAME\n",
       "0      yoplait berry delight                 DAIRY - YOGHURT\n",
       "1  destiny playstatpre order          PET NEEDS - DOG TREATS\n",
       "2               disney plush                   BABY - BASICS\n",
       "3         gold alula newborn                    BABY FORMULA\n",
       "4         rowy caramel fudge  CHEESE PRE-PACKED ENTERTAINING"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MILK - FLAVOURED              809\n",
       "DAIRY - MILK                  546\n",
       "LONGLIFE MILK & SOY DRINKS    423\n",
       "MILK ADDITIVES                291\n",
       "Name: SUBCAT_NAME, dtype: int64"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['SUBCAT_NAME'].str.contains(\"MILK\")]['SUBCAT_NAME'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    " counts=data['ARTICLE_NAME'].str.split(' ', expand=True).stack().value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts['item']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>Freq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dvd</td>\n",
       "      <td>4762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chicken</td>\n",
       "      <td>3620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>black</td>\n",
       "      <td>3403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>card</td>\n",
       "      <td>3306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cheese</td>\n",
       "      <td>3068</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      word  Freq\n",
       "0      dvd  4762\n",
       "1  chicken  3620\n",
       "2    black  3403\n",
       "3     card  3306\n",
       "4   cheese  3068"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts2=pd.DataFrame(counts)\n",
    "counts2=counts2.reset_index()\n",
    "counts2.columns=[\"word\",\"Freq\"]\n",
    "counts2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    58540.000000\n",
       "mean        12.026375\n",
       "std         76.567437\n",
       "min          1.000000\n",
       "25%          1.000000\n",
       "50%          1.000000\n",
       "75%          4.000000\n",
       "max       4762.000000\n",
       "Name: Freq, dtype: float64"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts2['Freq'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2288, 2)"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts3=counts2[counts2['Freq']>50]\n",
    "counts3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1175,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "counts3.to_csv(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/article_word_counts7.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2288"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(counts3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_prods=list(data['ARTICLE_NAME'])\n",
    "t=data['ARTICLE_NAME'].tolist()\n",
    "t2=t\n",
    "for i in range(len(t)):\n",
    "    t2[i]=t[i].split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-13 17:04:33,942 : INFO : collecting all words and their counts\n",
      "2017-11-13 17:04:33,944 : INFO : PROGRESS: at sentence #0, processed 0 words and 0 word types\n",
      "2017-11-13 17:04:34,044 : INFO : PROGRESS: at sentence #10000, processed 29571 words and 30908 word types\n",
      "2017-11-13 17:04:34,146 : INFO : PROGRESS: at sentence #20000, processed 59268 words and 52666 word types\n",
      "2017-11-13 17:04:34,248 : INFO : PROGRESS: at sentence #30000, processed 89023 words and 70940 word types\n",
      "2017-11-13 17:04:34,350 : INFO : PROGRESS: at sentence #40000, processed 118590 words and 87322 word types\n",
      "2017-11-13 17:04:34,454 : INFO : PROGRESS: at sentence #50000, processed 148040 words and 102325 word types\n",
      "2017-11-13 17:04:34,556 : INFO : PROGRESS: at sentence #60000, processed 177456 words and 116030 word types\n",
      "2017-11-13 17:04:34,658 : INFO : PROGRESS: at sentence #70000, processed 207196 words and 129392 word types\n",
      "2017-11-13 17:04:34,766 : INFO : PROGRESS: at sentence #80000, processed 236810 words and 141999 word types\n",
      "2017-11-13 17:04:34,866 : INFO : PROGRESS: at sentence #90000, processed 264693 words and 164093 word types\n",
      "2017-11-13 17:04:34,980 : INFO : PROGRESS: at sentence #100000, processed 292101 words and 187892 word types\n",
      "2017-11-13 17:04:35,083 : INFO : PROGRESS: at sentence #110000, processed 319516 words and 209048 word types\n",
      "2017-11-13 17:04:35,180 : INFO : PROGRESS: at sentence #120000, processed 346733 words and 228337 word types\n",
      "2017-11-13 17:04:35,279 : INFO : PROGRESS: at sentence #130000, processed 373877 words and 246395 word types\n",
      "2017-11-13 17:04:35,378 : INFO : PROGRESS: at sentence #140000, processed 401318 words and 263872 word types\n",
      "2017-11-13 17:04:35,477 : INFO : PROGRESS: at sentence #150000, processed 428663 words and 280608 word types\n",
      "2017-11-13 17:04:35,583 : INFO : PROGRESS: at sentence #160000, processed 456200 words and 296912 word types\n",
      "2017-11-13 17:04:35,688 : INFO : PROGRESS: at sentence #170000, processed 483457 words and 312330 word types\n",
      "2017-11-13 17:04:35,785 : INFO : PROGRESS: at sentence #180000, processed 510699 words and 327315 word types\n",
      "2017-11-13 17:04:35,820 : INFO : collected 332458 word types from a corpus of 520454 words (unigram + bigrams) and 183570 sentences\n",
      "2017-11-13 17:04:35,821 : INFO : using 332458 counts as vocab in Phrases<0 vocab, min_count=50, threshold=2, max_vocab_size=40000000>\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Phrases\n",
    "bigram = Phrases(t2, min_count=50, threshold=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\gensim\\models\\phrases.py:274: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    }
   ],
   "source": [
    "t3=[]\n",
    "for i in range(len(t2)):\n",
    "    sent=t2[i]\n",
    "    t3.append(bigram[sent])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['yoplait', 'berry', 'delight'],\n",
       " ['destiny', 'playstatpre', 'order'],\n",
       " ['disney', 'plush'],\n",
       " ['gold', 'alula', 'newborn'],\n",
       " ['rowy', 'caramel', 'fudge']]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t3[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',\\\n",
    "    level=logging.INFO)\n",
    "\n",
    "# Set values for various parameters\n",
    "num_features = 100    # Word vector dimensionality                      \n",
    "min_word_count = 50   # Minimum word count                        \n",
    "num_workers = 4       # Number of threads to run in parallel\n",
    "context = 9         # Context window size                                                                                    \n",
    "downsampling = 1e-3   # Downsample setting for frequent words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-13 17:04:43,794 : INFO : collecting all words and their counts\n",
      "2017-11-13 17:04:43,795 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-11-13 17:04:43,815 : INFO : PROGRESS: at sentence #10000, processed 37416 words, keeping 8742 word types\n",
      "2017-11-13 17:04:43,835 : INFO : PROGRESS: at sentence #20000, processed 74988 words, keeping 12950 word types\n",
      "2017-11-13 17:04:43,855 : INFO : PROGRESS: at sentence #30000, processed 112586 words, keeping 16177 word types\n",
      "2017-11-13 17:04:43,875 : INFO : PROGRESS: at sentence #40000, processed 150072 words, keeping 18882 word types\n",
      "2017-11-13 17:04:43,895 : INFO : PROGRESS: at sentence #50000, processed 187472 words, keeping 21304 word types\n",
      "2017-11-13 17:04:43,917 : INFO : PROGRESS: at sentence #60000, processed 224781 words, keeping 23374 word types\n",
      "2017-11-13 17:04:43,942 : INFO : PROGRESS: at sentence #70000, processed 262413 words, keeping 25415 word types\n",
      "2017-11-13 17:04:43,969 : INFO : PROGRESS: at sentence #80000, processed 300027 words, keeping 27313 word types\n",
      "2017-11-13 17:04:43,990 : INFO : PROGRESS: at sentence #90000, processed 336585 words, keeping 32071 word types\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-13 17:04:44,010 : INFO : PROGRESS: at sentence #100000, processed 372918 words, keeping 36653 word types\n",
      "2017-11-13 17:04:44,030 : INFO : PROGRESS: at sentence #110000, processed 409285 words, keeping 40394 word types\n",
      "2017-11-13 17:04:44,054 : INFO : PROGRESS: at sentence #120000, processed 445485 words, keeping 43536 word types\n",
      "2017-11-13 17:04:44,086 : INFO : PROGRESS: at sentence #130000, processed 481527 words, keeping 46364 word types\n",
      "2017-11-13 17:04:44,107 : INFO : PROGRESS: at sentence #140000, processed 517924 words, keeping 48936 word types\n",
      "2017-11-13 17:04:44,129 : INFO : PROGRESS: at sentence #150000, processed 554260 words, keeping 51380 word types\n",
      "2017-11-13 17:04:44,152 : INFO : PROGRESS: at sentence #160000, processed 590717 words, keeping 53804 word types\n",
      "2017-11-13 17:04:44,178 : INFO : PROGRESS: at sentence #170000, processed 626921 words, keeping 55987 word types\n",
      "2017-11-13 17:04:44,201 : INFO : PROGRESS: at sentence #180000, processed 663141 words, keeping 58107 word types\n",
      "2017-11-13 17:04:44,212 : INFO : collected 58837 word types from a corpus of 676075 raw words and 183570 sentences\n",
      "2017-11-13 17:04:44,213 : INFO : Loading a fresh vocabulary\n",
      "2017-11-13 17:04:44,265 : INFO : min_count=50 retains 2540 unique words (4% of original 58837, drops 56297)\n",
      "2017-11-13 17:04:44,267 : INFO : min_count=50 leaves 458085 word corpus (67% of original 676075, drops 217990)\n",
      "2017-11-13 17:04:44,291 : INFO : deleting the raw counts dictionary of 58837 items\n",
      "2017-11-13 17:04:44,296 : INFO : sample=0.001 downsamples 36 most-common words\n",
      "2017-11-13 17:04:44,297 : INFO : downsampling leaves estimated 441516 word corpus (96.4% of prior 458085)\n",
      "2017-11-13 17:04:44,298 : INFO : estimated required memory for 2540 words and 100 dimensions: 3302000 bytes\n",
      "2017-11-13 17:04:44,312 : INFO : resetting layer weights\n",
      "2017-11-13 17:04:44,371 : INFO : training model with 4 workers on 2540 vocabulary and 100 features, using sg=0 hs=0 sample=0.001 negative=5 window=9\n",
      "2017-11-13 17:04:45,407 : INFO : PROGRESS: at 29.12% examples, 655148 words/s, in_qsize 8, out_qsize 0\n",
      "2017-11-13 17:04:46,413 : INFO : PROGRESS: at 57.94% examples, 635982 words/s, in_qsize 7, out_qsize 0\n",
      "2017-11-13 17:04:47,433 : INFO : PROGRESS: at 88.00% examples, 645186 words/s, in_qsize 7, out_qsize 0\n",
      "2017-11-13 17:04:47,770 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-11-13 17:04:47,771 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-11-13 17:04:47,778 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-11-13 17:04:47,779 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-11-13 17:04:47,779 : INFO : training on 3380375 raw words (2207624 effective words) took 3.4s, 652353 effective words/s\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import word2vec\n",
    "print(\"Training model...\")\n",
    "w2v_model = word2vec.Word2Vec(t3, workers=num_workers, \\\n",
    "            size=num_features, min_count = min_word_count, \\\n",
    "            window = context, sample = downsampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-13 17:04:49,886 : INFO : precomputing L2-norms of word weight vectors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('chse', 0.8616633415222168),\n",
       " ('mainland_cheese', 0.8574714660644531),\n",
       " ('cheddar', 0.8551862239837646),\n",
       " ('ch', 0.8419789671897888),\n",
       " ('mainland', 0.8169941902160645),\n",
       " ('grated', 0.7922428250312805),\n",
       " ('cheese_slouse', 0.785186767578125),\n",
       " ('salami', 0.7840813398361206),\n",
       " ('brie', 0.7780658602714539),\n",
       " ('gouda', 0.7772737145423889)]"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_model.most_similar(\"cheese\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(183570, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARTICLE_NAME</th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yoplait berry delight</td>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>destiny playstatpre order</td>\n",
       "      <td>PET NEEDS - DOG TREATS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>disney plush</td>\n",
       "      <td>BABY - BASICS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gold alula newborn</td>\n",
       "      <td>BABY FORMULA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rowy caramel fudge</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                ARTICLE_NAME                     SUBCAT_NAME\n",
       "0      yoplait berry delight                 DAIRY - YOGHURT\n",
       "1  destiny playstatpre order          PET NEEDS - DOG TREATS\n",
       "2               disney plush                   BABY - BASICS\n",
       "3         gold alula newborn                    BABY FORMULA\n",
       "4         rowy caramel fudge  CHEESE PRE-PACKED ENTERTAINING"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2=data\n",
    "print(data2.shape)\n",
    "data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "120000\n",
      "130000\n",
      "140000\n",
      "150000\n",
      "160000\n",
      "170000\n",
      "180000\n"
     ]
    }
   ],
   "source": [
    "ind=[]\n",
    "dep=[]\n",
    "ex=[]\n",
    "word=[]\n",
    "x=t3\n",
    "t2=x\n",
    "y=data2['SUBCAT_NAME'].tolist()\n",
    "k=0\n",
    "m=0\n",
    "for i in range(len(x)):\n",
    "    t2[i]=x[i]\n",
    "    if(i%10000==0):\n",
    "        print(i)\n",
    "    #print(t2[i])\n",
    "    for j in range(len(t2[i])):\n",
    "        try:\n",
    "            ind.append((w2v_model[t2[i][j]]))\n",
    "            dep.append(y[i])\n",
    "            ex.append(data2['ARTICLE_NAME'][i])\n",
    "            word.append(t2[i][j])\n",
    "            m=m+1\n",
    "            #print(m)\n",
    "        except KeyError:\n",
    "            #print(\\its ok\\)\n",
    "            k=k+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>90</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.502269</td>\n",
       "      <td>0.065828</td>\n",
       "      <td>0.532839</td>\n",
       "      <td>-0.876406</td>\n",
       "      <td>-0.042052</td>\n",
       "      <td>-0.208942</td>\n",
       "      <td>-0.426184</td>\n",
       "      <td>0.461035</td>\n",
       "      <td>0.945602</td>\n",
       "      <td>0.214742</td>\n",
       "      <td>...</td>\n",
       "      <td>1.175375</td>\n",
       "      <td>-0.215694</td>\n",
       "      <td>-0.760396</td>\n",
       "      <td>0.294615</td>\n",
       "      <td>0.128994</td>\n",
       "      <td>-0.614231</td>\n",
       "      <td>-0.133625</td>\n",
       "      <td>-0.555341</td>\n",
       "      <td>-0.086292</td>\n",
       "      <td>0.309654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.627736</td>\n",
       "      <td>-0.113400</td>\n",
       "      <td>0.509135</td>\n",
       "      <td>-0.398738</td>\n",
       "      <td>-0.296475</td>\n",
       "      <td>0.163595</td>\n",
       "      <td>-0.656789</td>\n",
       "      <td>0.064620</td>\n",
       "      <td>0.628424</td>\n",
       "      <td>-0.282380</td>\n",
       "      <td>...</td>\n",
       "      <td>0.976393</td>\n",
       "      <td>-0.242951</td>\n",
       "      <td>-0.749223</td>\n",
       "      <td>0.369747</td>\n",
       "      <td>0.336394</td>\n",
       "      <td>-0.381900</td>\n",
       "      <td>0.028104</td>\n",
       "      <td>-0.491618</td>\n",
       "      <td>-0.303166</td>\n",
       "      <td>-0.256417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.249966</td>\n",
       "      <td>-0.197279</td>\n",
       "      <td>0.119820</td>\n",
       "      <td>-0.255025</td>\n",
       "      <td>0.279711</td>\n",
       "      <td>-0.285171</td>\n",
       "      <td>-0.348683</td>\n",
       "      <td>-0.050246</td>\n",
       "      <td>0.356724</td>\n",
       "      <td>0.118928</td>\n",
       "      <td>...</td>\n",
       "      <td>0.495194</td>\n",
       "      <td>0.111556</td>\n",
       "      <td>-0.284101</td>\n",
       "      <td>0.469044</td>\n",
       "      <td>0.164614</td>\n",
       "      <td>-0.334468</td>\n",
       "      <td>-0.006587</td>\n",
       "      <td>-0.322965</td>\n",
       "      <td>-0.071901</td>\n",
       "      <td>0.221618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.204293</td>\n",
       "      <td>-0.047774</td>\n",
       "      <td>0.692463</td>\n",
       "      <td>0.268272</td>\n",
       "      <td>-0.148196</td>\n",
       "      <td>0.284098</td>\n",
       "      <td>0.049278</td>\n",
       "      <td>0.356313</td>\n",
       "      <td>0.605597</td>\n",
       "      <td>-0.414332</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.103590</td>\n",
       "      <td>-0.549331</td>\n",
       "      <td>-0.670091</td>\n",
       "      <td>0.419861</td>\n",
       "      <td>-0.474965</td>\n",
       "      <td>-0.255029</td>\n",
       "      <td>-0.114241</td>\n",
       "      <td>-0.233007</td>\n",
       "      <td>-0.267427</td>\n",
       "      <td>-0.001939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.314717</td>\n",
       "      <td>-0.226887</td>\n",
       "      <td>0.325792</td>\n",
       "      <td>0.142807</td>\n",
       "      <td>-0.105028</td>\n",
       "      <td>0.041693</td>\n",
       "      <td>0.014319</td>\n",
       "      <td>0.413398</td>\n",
       "      <td>0.319911</td>\n",
       "      <td>-0.220551</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.134301</td>\n",
       "      <td>-0.196859</td>\n",
       "      <td>-0.522430</td>\n",
       "      <td>0.216962</td>\n",
       "      <td>-0.111961</td>\n",
       "      <td>-0.364099</td>\n",
       "      <td>-0.051591</td>\n",
       "      <td>-0.008306</td>\n",
       "      <td>-0.319499</td>\n",
       "      <td>0.140814</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4         5         6   \\\n",
       "0 -0.502269  0.065828  0.532839 -0.876406 -0.042052 -0.208942 -0.426184   \n",
       "1 -0.627736 -0.113400  0.509135 -0.398738 -0.296475  0.163595 -0.656789   \n",
       "2 -0.249966 -0.197279  0.119820 -0.255025  0.279711 -0.285171 -0.348683   \n",
       "3  0.204293 -0.047774  0.692463  0.268272 -0.148196  0.284098  0.049278   \n",
       "4  0.314717 -0.226887  0.325792  0.142807 -0.105028  0.041693  0.014319   \n",
       "\n",
       "         7         8         9     ...           90        91        92  \\\n",
       "0  0.461035  0.945602  0.214742    ...     1.175375 -0.215694 -0.760396   \n",
       "1  0.064620  0.628424 -0.282380    ...     0.976393 -0.242951 -0.749223   \n",
       "2 -0.050246  0.356724  0.118928    ...     0.495194  0.111556 -0.284101   \n",
       "3  0.356313  0.605597 -0.414332    ...    -0.103590 -0.549331 -0.670091   \n",
       "4  0.413398  0.319911 -0.220551    ...    -0.134301 -0.196859 -0.522430   \n",
       "\n",
       "         93        94        95        96        97        98        99  \n",
       "0  0.294615  0.128994 -0.614231 -0.133625 -0.555341 -0.086292  0.309654  \n",
       "1  0.369747  0.336394 -0.381900  0.028104 -0.491618 -0.303166 -0.256417  \n",
       "2  0.469044  0.164614 -0.334468 -0.006587 -0.322965 -0.071901  0.221618  \n",
       "3  0.419861 -0.474965 -0.255029 -0.114241 -0.233007 -0.267427 -0.001939  \n",
       "4  0.216962 -0.111961 -0.364099 -0.051591 -0.008306 -0.319499  0.140814  \n",
       "\n",
       "[5 rows x 100 columns]"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt=0\n",
    "tt=pd.DataFrame(ind)\n",
    "tt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([            0,             1,             2,             3,\n",
       "                   4,             5,             6,             7,\n",
       "                   8,             9,\n",
       "       ...\n",
       "                  96,            97,            98,            99,\n",
       "            'subcat',     'article',        'word',      'cheese',\n",
       "       'canned_fish',       'chips'],\n",
       "      dtype='object', length=106)"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "tt['subcat']=dep\n",
    "tt['article']=ex\n",
    "tt['word']=word\n",
    "tt['cheese']=np.where(((tt['subcat']==\"DAIRY - CHEESE\") | (tt['subcat']==\"PASTA SAUCE & CHEESE\") | (tt['subcat']==\"CHEESE PRE-PACKED ENTERTAINING\") | (tt['subcat']==\"CHEESE PRE-PACKED COOKING\") | (tt['subcat']==\"CHEESE PRE-PACKED COOKING\") | (tt['subcat']==\"CHEESE BULK\")),1.0,0.0)\n",
    "tt['canned_fish']=np.where(tt['subcat']==\"CANNED FISH\",1.0,0.0)\n",
    "tt['chips']=np.where(((tt['subcat']==\"CHIPS - SINGLE SERVE\") | (tt['subcat']==\"CHIPS - SHARING\") | (tt['subcat']==\"CHIPS - MULTIPACKS\")),1.0,0.0)\n",
    "tt.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "      <th>subcat</th>\n",
       "      <th>article</th>\n",
       "      <th>word</th>\n",
       "      <th>cheese</th>\n",
       "      <th>canned_fish</th>\n",
       "      <th>chips</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.502269</td>\n",
       "      <td>0.065828</td>\n",
       "      <td>0.532839</td>\n",
       "      <td>-0.876406</td>\n",
       "      <td>-0.042052</td>\n",
       "      <td>-0.208942</td>\n",
       "      <td>-0.426184</td>\n",
       "      <td>0.461035</td>\n",
       "      <td>0.945602</td>\n",
       "      <td>0.214742</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.133625</td>\n",
       "      <td>-0.555341</td>\n",
       "      <td>-0.086292</td>\n",
       "      <td>0.309654</td>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "      <td>yoplait berry delight</td>\n",
       "      <td>yoplait</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.627736</td>\n",
       "      <td>-0.113400</td>\n",
       "      <td>0.509135</td>\n",
       "      <td>-0.398738</td>\n",
       "      <td>-0.296475</td>\n",
       "      <td>0.163595</td>\n",
       "      <td>-0.656789</td>\n",
       "      <td>0.064620</td>\n",
       "      <td>0.628424</td>\n",
       "      <td>-0.282380</td>\n",
       "      <td>...</td>\n",
       "      <td>0.028104</td>\n",
       "      <td>-0.491618</td>\n",
       "      <td>-0.303166</td>\n",
       "      <td>-0.256417</td>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "      <td>yoplait berry delight</td>\n",
       "      <td>berry</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.249966</td>\n",
       "      <td>-0.197279</td>\n",
       "      <td>0.119820</td>\n",
       "      <td>-0.255025</td>\n",
       "      <td>0.279711</td>\n",
       "      <td>-0.285171</td>\n",
       "      <td>-0.348683</td>\n",
       "      <td>-0.050246</td>\n",
       "      <td>0.356724</td>\n",
       "      <td>0.118928</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.006587</td>\n",
       "      <td>-0.322965</td>\n",
       "      <td>-0.071901</td>\n",
       "      <td>0.221618</td>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "      <td>yoplait berry delight</td>\n",
       "      <td>delight</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.204293</td>\n",
       "      <td>-0.047774</td>\n",
       "      <td>0.692463</td>\n",
       "      <td>0.268272</td>\n",
       "      <td>-0.148196</td>\n",
       "      <td>0.284098</td>\n",
       "      <td>0.049278</td>\n",
       "      <td>0.356313</td>\n",
       "      <td>0.605597</td>\n",
       "      <td>-0.414332</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.114241</td>\n",
       "      <td>-0.233007</td>\n",
       "      <td>-0.267427</td>\n",
       "      <td>-0.001939</td>\n",
       "      <td>BABY - BASICS</td>\n",
       "      <td>disney plush</td>\n",
       "      <td>disney</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.314717</td>\n",
       "      <td>-0.226887</td>\n",
       "      <td>0.325792</td>\n",
       "      <td>0.142807</td>\n",
       "      <td>-0.105028</td>\n",
       "      <td>0.041693</td>\n",
       "      <td>0.014319</td>\n",
       "      <td>0.413398</td>\n",
       "      <td>0.319911</td>\n",
       "      <td>-0.220551</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.051591</td>\n",
       "      <td>-0.008306</td>\n",
       "      <td>-0.319499</td>\n",
       "      <td>0.140814</td>\n",
       "      <td>BABY - BASICS</td>\n",
       "      <td>disney plush</td>\n",
       "      <td>plush</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0 -0.502269  0.065828  0.532839 -0.876406 -0.042052 -0.208942 -0.426184   \n",
       "1 -0.627736 -0.113400  0.509135 -0.398738 -0.296475  0.163595 -0.656789   \n",
       "2 -0.249966 -0.197279  0.119820 -0.255025  0.279711 -0.285171 -0.348683   \n",
       "3  0.204293 -0.047774  0.692463  0.268272 -0.148196  0.284098  0.049278   \n",
       "4  0.314717 -0.226887  0.325792  0.142807 -0.105028  0.041693  0.014319   \n",
       "\n",
       "          7         8         9  ...          96        97        98  \\\n",
       "0  0.461035  0.945602  0.214742  ...   -0.133625 -0.555341 -0.086292   \n",
       "1  0.064620  0.628424 -0.282380  ...    0.028104 -0.491618 -0.303166   \n",
       "2 -0.050246  0.356724  0.118928  ...   -0.006587 -0.322965 -0.071901   \n",
       "3  0.356313  0.605597 -0.414332  ...   -0.114241 -0.233007 -0.267427   \n",
       "4  0.413398  0.319911 -0.220551  ...   -0.051591 -0.008306 -0.319499   \n",
       "\n",
       "         99           subcat                article     word  cheese  \\\n",
       "0  0.309654  DAIRY - YOGHURT  yoplait berry delight  yoplait     0.0   \n",
       "1 -0.256417  DAIRY - YOGHURT  yoplait berry delight    berry     0.0   \n",
       "2  0.221618  DAIRY - YOGHURT  yoplait berry delight  delight     0.0   \n",
       "3 -0.001939    BABY - BASICS           disney plush   disney     0.0   \n",
       "4  0.140814    BABY - BASICS           disney plush    plush     0.0   \n",
       "\n",
       "   canned_fish  chips  \n",
       "0          0.0    0.0  \n",
       "1          0.0    0.0  \n",
       "2          0.0    0.0  \n",
       "3          0.0    0.0  \n",
       "4          0.0    0.0  \n",
       "\n",
       "[5 rows x 106 columns]"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tt2=pd.pivot_table(tt.drop(['subcat','word','cheese','canned_fish','chips'],axis=1),index=['article'],aggfunc=np.mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158292, 102)"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt2=tt2.reset_index()\n",
    "tt3=tt[['article','subcat']]\n",
    "tt4=tt3.drop_duplicates()\n",
    "tt5=pd.merge(tt2,tt4)\n",
    "tt5.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tt_word=pd.merge(tt,counts2,on='word',how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(430524, 107)"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt_word.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(458085, 106)"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:1: FutureWarning: sort(columns=....) is deprecated, use sort_values(by=.....)\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "tt_word=tt_word.sort(['article'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tt_word=tt_word.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(430524, 107)"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt_word=tt_word.drop(['Freq'],axis=1)\n",
    "tt_word.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(430524, 106)"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt_word=tt_word.drop(['index'],axis=1)\n",
    "tt_word.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tt_word2=pd.pivot_table(tt.drop(['subcat','word','cheese','canned_fish','chips'],axis=1),index=['article'],aggfunc=np.mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(155474, 102)"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt_word2=tt_word2.reset_index()\n",
    "tt_word3=tt_word[['article','subcat']]\n",
    "tt_word4=tt_word3.drop_duplicates()\n",
    "tt_word5=pd.merge(tt_word2,tt_word4)\n",
    "tt_word5.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tt6=tt5.append(tt_word5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#tt6=tt5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(313766, 102)"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt6.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tt6['cheese']=np.where(((tt6['subcat']==\"DAIRY - CHEESE\") | (tt6['subcat']==\"PASTA SAUCE & CHEESE\") | (tt6['subcat']==\"CHEESE PRE-PACKED ENTERTAINING\") | (tt6['subcat']==\"CHEESE PRE-PACKED COOKING\") | (tt6['subcat']==\"CHEESE PRE-PACKED COOKING\") | (tt6['subcat']==\"CHEESE BULK\")),1.0,0.0)\n",
    "tt6['canned_fish']=np.where(tt6['subcat']==\"CANNED FISH\",1.0,0.0)\n",
    "tt6['chips']=np.where(((tt6['subcat']==\"CHIPS - SINGLE SERVE\") | (tt6['subcat']==\"CHIPS - SHARING\") | (tt6['subcat']==\"CHIPS - MULTIPACKS\")),1.0,0.0)\n",
    "tt6['final_subcat']=np.where(tt6['cheese']==1.0,1,np.where(tt6['canned_fish']==1.0,2,np.where(tt6['chips']==1.0,3,0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([     'article',              0,              1,              2,\n",
       "                    3,              4,              5,              6,\n",
       "                    7,              8,\n",
       "       ...\n",
       "                   95,             96,             97,             98,\n",
       "                   99,       'subcat',       'cheese',  'canned_fish',\n",
       "              'chips', 'final_subcat'],\n",
       "      dtype='object', length=106)"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt6.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>95</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "      <th>subcat</th>\n",
       "      <th>cheese</th>\n",
       "      <th>canned_fish</th>\n",
       "      <th>chips</th>\n",
       "      <th>final_subcat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aa air freshener vanilla</td>\n",
       "      <td>-0.174764</td>\n",
       "      <td>-0.215149</td>\n",
       "      <td>0.324503</td>\n",
       "      <td>-0.244157</td>\n",
       "      <td>-0.143780</td>\n",
       "      <td>-0.090593</td>\n",
       "      <td>-0.252002</td>\n",
       "      <td>0.354626</td>\n",
       "      <td>0.304842</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.327167</td>\n",
       "      <td>0.012143</td>\n",
       "      <td>-0.161202</td>\n",
       "      <td>-0.103043</td>\n",
       "      <td>0.060150</td>\n",
       "      <td>unknown</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aa attum flour</td>\n",
       "      <td>0.090097</td>\n",
       "      <td>-0.187574</td>\n",
       "      <td>0.084583</td>\n",
       "      <td>-0.257561</td>\n",
       "      <td>0.049600</td>\n",
       "      <td>-0.161327</td>\n",
       "      <td>-0.113648</td>\n",
       "      <td>0.135219</td>\n",
       "      <td>0.152690</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.299314</td>\n",
       "      <td>-0.036575</td>\n",
       "      <td>-0.162999</td>\n",
       "      <td>-0.099049</td>\n",
       "      <td>0.164816</td>\n",
       "      <td>unknown</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aa order required</td>\n",
       "      <td>0.124038</td>\n",
       "      <td>-0.300707</td>\n",
       "      <td>0.124215</td>\n",
       "      <td>-0.083251</td>\n",
       "      <td>-0.061721</td>\n",
       "      <td>-0.042778</td>\n",
       "      <td>-0.046722</td>\n",
       "      <td>0.178745</td>\n",
       "      <td>0.261352</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.204476</td>\n",
       "      <td>0.037098</td>\n",
       "      <td>-0.084059</td>\n",
       "      <td>-0.130874</td>\n",
       "      <td>0.157403</td>\n",
       "      <td>unknown</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aapri scrub facial original</td>\n",
       "      <td>0.126395</td>\n",
       "      <td>-0.359556</td>\n",
       "      <td>0.093018</td>\n",
       "      <td>-0.063419</td>\n",
       "      <td>-0.173237</td>\n",
       "      <td>-0.147554</td>\n",
       "      <td>-0.192202</td>\n",
       "      <td>0.122712</td>\n",
       "      <td>0.236076</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.235825</td>\n",
       "      <td>0.079171</td>\n",
       "      <td>-0.119677</td>\n",
       "      <td>-0.034439</td>\n",
       "      <td>0.139863</td>\n",
       "      <td>unknown</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aashiayana chicken tikka marsala</td>\n",
       "      <td>0.414469</td>\n",
       "      <td>-0.818314</td>\n",
       "      <td>0.238988</td>\n",
       "      <td>-0.557760</td>\n",
       "      <td>0.500375</td>\n",
       "      <td>-0.058706</td>\n",
       "      <td>-0.174958</td>\n",
       "      <td>-0.242977</td>\n",
       "      <td>0.215147</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.189325</td>\n",
       "      <td>0.411273</td>\n",
       "      <td>-0.418959</td>\n",
       "      <td>-0.368676</td>\n",
       "      <td>0.549393</td>\n",
       "      <td>FREEZER - MEALS</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            article         0         1         2         3  \\\n",
       "0          aa air freshener vanilla -0.174764 -0.215149  0.324503 -0.244157   \n",
       "1                    aa attum flour  0.090097 -0.187574  0.084583 -0.257561   \n",
       "2                 aa order required  0.124038 -0.300707  0.124215 -0.083251   \n",
       "3       aapri scrub facial original  0.126395 -0.359556  0.093018 -0.063419   \n",
       "4  aashiayana chicken tikka marsala  0.414469 -0.818314  0.238988 -0.557760   \n",
       "\n",
       "          4         5         6         7         8      ...             95  \\\n",
       "0 -0.143780 -0.090593 -0.252002  0.354626  0.304842      ...      -0.327167   \n",
       "1  0.049600 -0.161327 -0.113648  0.135219  0.152690      ...      -0.299314   \n",
       "2 -0.061721 -0.042778 -0.046722  0.178745  0.261352      ...      -0.204476   \n",
       "3 -0.173237 -0.147554 -0.192202  0.122712  0.236076      ...      -0.235825   \n",
       "4  0.500375 -0.058706 -0.174958 -0.242977  0.215147      ...      -0.189325   \n",
       "\n",
       "         96        97        98        99           subcat  cheese  \\\n",
       "0  0.012143 -0.161202 -0.103043  0.060150          unknown     0.0   \n",
       "1 -0.036575 -0.162999 -0.099049  0.164816          unknown     0.0   \n",
       "2  0.037098 -0.084059 -0.130874  0.157403          unknown     0.0   \n",
       "3  0.079171 -0.119677 -0.034439  0.139863          unknown     0.0   \n",
       "4  0.411273 -0.418959 -0.368676  0.549393  FREEZER - MEALS     0.0   \n",
       "\n",
       "   canned_fish  chips  final_subcat  \n",
       "0          0.0    0.0             0  \n",
       "1          0.0    0.0             0  \n",
       "2          0.0    0.0             0  \n",
       "3          0.0    0.0             0  \n",
       "4          0.0    0.0             0  \n",
       "\n",
       "[5 rows x 106 columns]"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt6.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.randint(0,5,1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subcat</th>\n",
       "      <th>make_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>unknown</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unknown</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unknown</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unknown</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FREEZER - MEALS</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>FREEZER - MEALS</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>unknown</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>HEALTH FOODS</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>HEALTH FOODS</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>unknown</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>unknown</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             subcat  make_code\n",
       "0           unknown        110\n",
       "1           unknown        110\n",
       "2           unknown        110\n",
       "3           unknown        110\n",
       "4   FREEZER - MEALS         53\n",
       "5   FREEZER - MEALS         53\n",
       "6           unknown        110\n",
       "7      HEALTH FOODS         66\n",
       "8      HEALTH FOODS         66\n",
       "9           unknown        110\n",
       "10          unknown        110"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "lb_make = LabelEncoder()\n",
    "tt6[\"make_code\"] = lb_make.fit_transform(tt6[\"subcat\"])\n",
    "tt6[[\"subcat\", \"make_code\"]].head(11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(313766, 101)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(313766,)"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X=tt6.iloc[:,0:(num_features+1)]\n",
    "print(X.shape)\n",
    "y=tt6['make_code']\n",
    "y.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# X=tt6.iloc[:,1:(num_features+1)]\n",
    "# print(X.shape)\n",
    "# y=tt6['subcat']\n",
    "# y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed(1)\n",
    "from sklearn.cross_validation import train_test_split\n",
    "x_train, x_test, y_train, y_test = train_test_split(X,y,test_size=0.3,random_state=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from sklearn.linear_model import LogisticRegression\n",
    "# lm=LogisticRegression()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# lm.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pred=lm.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pd.Series(pred).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# correct=np.where(pred==y_test,1,0)\n",
    "# np.mean(correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27787      21\n",
       "154326     82\n",
       "138253    110\n",
       "33549     110\n",
       "16984     110\n",
       "Name: make_code, dtype: int64"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed(1)\n",
    "from sklearn.cross_validation import train_test_split\n",
    "x_train, x_test, y_train, y_test = train_test_split(X,y,test_size=0.3,random_state=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train=np.array(x_train)\n",
    "y_train=np.array(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train=np.array(x_train.drop(['article'],axis=1))\n",
    "y_train=np.array(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "111"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "nb_classes = len(set(tt6['make_code']))\n",
    "\n",
    "def indices_to_one_hot(data, nb_classes):\n",
    "    \"\"\"Convert an iterable of indices to one-hot encoded labels.\"\"\"\n",
    "    targets = np.array(data).reshape(-1)\n",
    "    return np.eye(nb_classes)[targets]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "111"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train=np.array(x_train.drop(['article'],axis=1))\n",
    "y_train=np.array(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 63,  64, 108], dtype=int64)"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_train2=indices_to_one_hot(y_train,nb_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_test=np.array(x_test.drop(['article'],axis=1))\n",
    "y_test=np.array(y_test)\n",
    "y_test2=indices_to_one_hot(y_test,nb_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#y_test2=indices_to_one_hot(y_test1,nb_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:7: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1000, kernel_initializer=\"glorot_uniform\", activation=\"relu\", input_dim=100)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 219636 samples, validate on 94130 samples\n",
      "Epoch 1/10\n",
      "219636/219636 [==============================] - 20s - loss: 2.2293 - acc: 0.5506 - val_loss: 1.6701 - val_acc: 0.5923\n",
      "Epoch 2/10\n",
      "219636/219636 [==============================] - 19s - loss: 1.6442 - acc: 0.5934 - val_loss: 1.5130 - val_acc: 0.6203\n",
      "Epoch 3/10\n",
      "219636/219636 [==============================] - 19s - loss: 1.5454 - acc: 0.6100 - val_loss: 1.4459 - val_acc: 0.6310\n",
      "Epoch 4/10\n",
      "219636/219636 [==============================] - 19s - loss: 1.4929 - acc: 0.6211 - val_loss: 1.4083 - val_acc: 0.6389\n",
      "Epoch 5/10\n",
      "219636/219636 [==============================] - 19s - loss: 1.4576 - acc: 0.6272 - val_loss: 1.3765 - val_acc: 0.6449\n",
      "Epoch 6/10\n",
      "219636/219636 [==============================] - 19s - loss: 1.4301 - acc: 0.6327 - val_loss: 1.3545 - val_acc: 0.6483\n",
      "Epoch 7/10\n",
      "219636/219636 [==============================] - 19s - loss: 1.4099 - acc: 0.6375 - val_loss: 1.3361 - val_acc: 0.6519\n",
      "Epoch 8/10\n",
      "219636/219636 [==============================] - 20s - loss: 1.3910 - acc: 0.6408 - val_loss: 1.3185 - val_acc: 0.6542\n",
      "Epoch 9/10\n",
      "219636/219636 [==============================] - 19s - loss: 1.3724 - acc: 0.6426 - val_loss: 1.3048 - val_acc: 0.6566\n",
      "Epoch 10/10\n",
      "219636/219636 [==============================] - 20s - loss: 1.3600 - acc: 0.6462 - val_loss: 1.2928 - val_acc: 0.6589\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x192cb3e3ef0>"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seed(1)\n",
    "from keras.models import *\n",
    "from keras.layers import *\n",
    "# create model\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Dense(1000, input_dim=num_features, activation='relu',init='glorot_uniform'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(50, activation='relu'))\n",
    "model.add(Dense(nb_classes, activation='softmax'))\n",
    "# Compile model\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# Fit the model\n",
    "model.fit(X_train, y_train2, epochs=10, batch_size=1024,validation_data=(X_test, y_test2),shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 575,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# X_test1=X_test\n",
    "# y_test1=y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions = model.predict(X_train)\n",
    "predictions2=pd.DataFrame(predictions)\n",
    "predictions3=np.argmax(predictions,axis=1)\n",
    "X_train2=pd.DataFrame(X_train)\n",
    "X_train2['prediction']=predictions3\n",
    "X_train2['final_subcat']=y_train\n",
    "X_train2.index=x_train.index\n",
    "X_train2['article']=x_train['article']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([110,  64, 108,  94,   4], dtype=int64)"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions3[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>prediction</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>101</th>\n",
       "      <th>102</th>\n",
       "      <th>103</th>\n",
       "      <th>104</th>\n",
       "      <th>105</th>\n",
       "      <th>106</th>\n",
       "      <th>107</th>\n",
       "      <th>108</th>\n",
       "      <th>109</th>\n",
       "      <th>110</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>final_subcat</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>402</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>20</td>\n",
       "      <td>19</td>\n",
       "      <td>1057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>442</td>\n",
       "      <td>31</td>\n",
       "      <td>20</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>242</td>\n",
       "      <td>32</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>562</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>986</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>271</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>195</td>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>53</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>481</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>85</td>\n",
       "      <td>314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>283</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>280</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>426</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>147</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>741</td>\n",
       "      <td>0</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1452</td>\n",
       "      <td>516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>226</td>\n",
       "      <td>111</td>\n",
       "      <td>126</td>\n",
       "      <td>42</td>\n",
       "      <td>226</td>\n",
       "      <td>12</td>\n",
       "      <td>172</td>\n",
       "      <td>37</td>\n",
       "      <td>34</td>\n",
       "      <td>257</td>\n",
       "      <td>...</td>\n",
       "      <td>83</td>\n",
       "      <td>30</td>\n",
       "      <td>70</td>\n",
       "      <td>105</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>125</td>\n",
       "      <td>166</td>\n",
       "      <td>106225</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>111 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "prediction    0    1    2    3    4    5    6    7    8    9     ...    101  \\\n",
       "final_subcat                                                     ...          \n",
       "0             402    0    2    0    7    0    0    0    5   12   ...      2   \n",
       "1               0  442   31   20   35    0    0    0    0    0   ...      0   \n",
       "2               0   10  242   32    6    0    0    0    0    0   ...      0   \n",
       "3               0    3    0  562    0    0    0    0    0    0   ...      0   \n",
       "4               1    5    9    1  986   12    0    0    2    0   ...      0   \n",
       "5               0    0    4   12   12   51    0    0    0    0   ...      0   \n",
       "6               0    0    0    0    0    0  271   34    0    0   ...      0   \n",
       "7               2    0    0    0    0    0  195   58    0    0   ...      0   \n",
       "8               6    0    0    0    3    0    0    0  100   53   ...      0   \n",
       "9               0    0    0    0    0    0    0    0    2  481   ...      0   \n",
       "10              5    0    0    0    0    0    0    0   16    8   ...      0   \n",
       "11              0    0    0    0   16    0    0    0    2   10   ...      0   \n",
       "12              0    0    0    0   13    0    0    0    0    1   ...      0   \n",
       "13              0    0    0    0    2    0    0    0    0   31   ...      1   \n",
       "14             10    0    2    0    1    0    0    0    0    0   ...      0   \n",
       "15              0    0    0    0   13    0    0    0    0    0   ...      0   \n",
       "16              2    0    2    0   16    0    0    0    0    0   ...      3   \n",
       "17              0    0    0    0    4    0    0    0    0    0   ...      0   \n",
       "18              0    2    0    0    5    0    0    0    0    0   ...      0   \n",
       "19              1    0    0    0    7    0    0    0    2    2   ...      0   \n",
       "20              0    0    0    0    2    0    0    0    9    7   ...      0   \n",
       "21              5    0    0    0    1    0    0    0   13    4   ...      0   \n",
       "22              0    2    0    0    3    0    0    0    0    0   ...      0   \n",
       "23              0    0    0    0    0    0    0    0    0    7   ...      0   \n",
       "24              0    0    0    0    0    0    0    0    0    4   ...      0   \n",
       "25              0    0    0    0    2    0    0    0    0    2   ...      0   \n",
       "26              0    0    0    0    0    0    0    0    0   12   ...      0   \n",
       "27              0    2    0    0    0    0    0    0    0    0   ...      0   \n",
       "28              0    0    0    0    1    0    0    0    0    3   ...      1   \n",
       "29              0    0    0    0    0    0    0    0    0    2   ...      0   \n",
       "...           ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   ...    ...   \n",
       "81              0    0    0    2    1    0    2    0    0    3   ...      0   \n",
       "82              0    0    0    0    7    0    2    2    0    0   ...      0   \n",
       "83              0    0    0    0    0    0    3    0    0    1   ...      0   \n",
       "84              0    0    0    0    0    0    0    0    0    0   ...      0   \n",
       "85              0    0    0    0    5    0   15    0    0    0   ...      0   \n",
       "86              0    0    0    0    0    0    0    0    0    0   ...      0   \n",
       "87              0    0    0    0    0    0    0    0    0    0   ...      0   \n",
       "88              1    0    0    0    0    0    2    0    0    0   ...      0   \n",
       "89              0    0    0    0    0    0    0    0    0    0   ...      0   \n",
       "90              0    0    0    0    6    0    0    0    4    0   ...      0   \n",
       "91              0    1    0    0    0    0    0    0    0   38   ...      0   \n",
       "92              7    0    0    0    2    0    0    0    0    0   ...      0   \n",
       "93              2    0    0    0    0    0    0    0    0    0   ...      0   \n",
       "94             29    0    0    0   19    0    0    0    0    0   ...      0   \n",
       "95              0    0    0    0    2    0    4    0    2    2   ...      0   \n",
       "96              0    0    0    0    0    0    0    0    0    0   ...      7   \n",
       "97              0    0    0    0    0    0    0    0    0    0   ...      0   \n",
       "98              0    0    0    0    1    0    0    0    0    0   ...     49   \n",
       "99              0    0    0    0    1    0    0    0    0    0   ...     18   \n",
       "100             0    0    0    0    0    0    0    0    0    0   ...      9   \n",
       "101             0    0    0    0    0    0    0    0    0    0   ...    283   \n",
       "102             3    0    0    0    5    0    0    0    0    0   ...      0   \n",
       "103             0    0    0    0    2    0    0    0    0    0   ...      1   \n",
       "104             0    0    0    0    0    0    0    0    0    0   ...      1   \n",
       "105             0    0    0    0    3    0    0    0    0    1   ...      0   \n",
       "106             0    0    0    0    0    0    0    0    0    0   ...      0   \n",
       "107             0    0    0    0    3    0    0    0    0    0   ...      0   \n",
       "108             1    0    0    0    0    0    0    0    0    0   ...      0   \n",
       "109             2    0    0    0    4    0    0    0    0    0   ...      0   \n",
       "110           226  111  126   42  226   12  172   37   34  257   ...     83   \n",
       "\n",
       "prediction    102  103  104  105  106  107  108   109     110  \n",
       "final_subcat                                                   \n",
       "0               6    0    3    0    0    9   20    19    1057  \n",
       "1               0    0    0    0    0    0    0     0    1903  \n",
       "2               0    0    0    0    0    0    0     0     332  \n",
       "3               0    0    0    0    0    0    0     0     122  \n",
       "4               0    1    0    0    0    0    0     0     433  \n",
       "5               0    0    0    0    0    0    0     0      78  \n",
       "6               0    0    0    0    0    0    0     0     336  \n",
       "7               0    0    0    0    0    0    0     0     416  \n",
       "8               0    3    2    0    0    0    0     0     465  \n",
       "9               0    9    1    0    0    2    0     2     936  \n",
       "10              0    3    0    0    0    0    0     0     362  \n",
       "11              0   11    0    0    0    0    1     1     255  \n",
       "12              0    1    0    0    0    0    0     0     135  \n",
       "13              0    0    0    0    0   10    0     4    1027  \n",
       "14              0    0    0    0    0    0    2     0     351  \n",
       "15              0    0    6    0    0    0    0     0     257  \n",
       "16              0    0    0    0    0    0    0     0     475  \n",
       "17              0    0    0    0    0    0    0     2     254  \n",
       "18              0    0    0    0    0    0    0     0     218  \n",
       "19              0    3    6    0    0    0    0     0     822  \n",
       "20              0    0    0    0    0    0    0     0     203  \n",
       "21              0    0    0    0    0    0    0     0     694  \n",
       "22              0    0    0    0    0    4    0    39     534  \n",
       "23              0    0    0    0    0    0    0     0     295  \n",
       "24              0    0    1    0    0    3    0     4     415  \n",
       "25              0    0    0    0    0    0    0     2     117  \n",
       "26              0    1    2    0    0    0    0     1     941  \n",
       "27              0   11    0    0    0    2    0     3     330  \n",
       "28              0    0    0    0    0    0    0     4     785  \n",
       "29              0    0    0    0    0    0    0     0     134  \n",
       "...           ...  ...  ...  ...  ...  ...  ...   ...     ...  \n",
       "81              0    0    0    0    0    0    2     0     129  \n",
       "82              0    0    0    0    0    0    0     0     150  \n",
       "83              0    0    0    0    0    0    3     0     242  \n",
       "84              0   11    0    0    0    0    0     0     123  \n",
       "85              0    0    0    0    0    0    4     0     141  \n",
       "86              0    0    0    0    0    0    0     0     126  \n",
       "87              0    0    0    0    0    0    0     0      85  \n",
       "88              0    0    0    0    0    0    2     0     236  \n",
       "89              0    0    0    0    0    0    0     0      48  \n",
       "90              0    0    1    0    0    0    4     0     567  \n",
       "91              0    1   17    0    0    2    0     2    1281  \n",
       "92              0    0    0    0    0    0    2     0     122  \n",
       "93              0    0    0    0    0    0    0     0     286  \n",
       "94              0    3    1    0    0    0    1     0     368  \n",
       "95              0    1    0    0    0    0    0     0     591  \n",
       "96              0    0    0    0    0    2    0     2     421  \n",
       "97              0    0    0    0    0    3    0     0     238  \n",
       "98              0    0    0    0    0    0    0     2     135  \n",
       "99              0    0    0    0    0    4    0     0     281  \n",
       "100             0    2    0    0    0    0    0    85     314  \n",
       "101             0    0    0    0    0    0    0     3     267  \n",
       "102            19    0    0    0    0    0    0     0     136  \n",
       "103             0  280    0    0    0    0    0     0     108  \n",
       "104             0    0  426    0    0    1    0     2     299  \n",
       "105             0    0    2    0    9    0    0     0      95  \n",
       "106             0    0    0    0   33    0    0     0      50  \n",
       "107             0    1    2    0    0  147    0     0     132  \n",
       "108             0    0    0    0    0    0  741     0      95  \n",
       "109             0    1    2    0    0    0    0  1452     516  \n",
       "110            30   70  105    0    2   25  125   166  106225  \n",
       "\n",
       "[111 rows x 100 columns]"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(X_train2['final_subcat'],X_train2['prediction'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions = model.predict(X_test)\n",
    "predictions2=pd.DataFrame(predictions)\n",
    "predictions3=np.argmax(predictions,axis=1)\n",
    "X_test2=pd.DataFrame(X_test)\n",
    "X_test2['prediction']=predictions3\n",
    "X_test2['final_subcat']=y_test\n",
    "X_test2.index=x_test.index\n",
    "X_test2['article']=x_test['article']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([     'article',              0,              1,              2,\n",
       "                    3,              4,              5,              6,\n",
       "                    7,              8,\n",
       "       ...\n",
       "                   96,             97,             98,             99,\n",
       "             'subcat',       'cheese',  'canned_fish',        'chips',\n",
       "       'final_subcat',    'make_code'],\n",
       "      dtype='object', length=107)"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt6.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "subcat_index_mapping=tt6[['make_code','subcat']].drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subcat_index_mapping=subcat_index_mapping.reset_index()\n",
    "subcat_index_mapping=subcat_index_mapping.drop(['index'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>make_code</th>\n",
       "      <th>subcat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1</td>\n",
       "      <td>BABY - BASICS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    make_code         subcat\n",
       "28          1  BABY - BASICS"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subcat_index_mapping[subcat_index_mapping['make_code']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.658918516944651"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test2['correct']=np.where(X_test2['final_subcat']==X_test2['prediction'],1,0)\n",
    "np.mean(X_test2['correct'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test2=X_test2.reset_index()\n",
    "X_test2=X_test2.drop(['index'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RangeIndex(start=0, stop=94130, step=1)"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test2.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_test2=pd.merge(X_test2,subcat_index_mapping,left_on='final_subcat',right_on='make_code',how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([             0,              1,              2,              3,\n",
       "                    4,              5,              6,              7,\n",
       "                    8,              9,\n",
       "       ...\n",
       "                   96,             97,             98,             99,\n",
       "         'prediction', 'final_subcat',      'article',      'correct',\n",
       "          'make_code',       'subcat'],\n",
       "      dtype='object', length=106)"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test2.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test2=X_test2.rename(columns={\"subcat\":\"final_subcat_name\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test2=pd.merge(X_test2,subcat_index_mapping,left_on='prediction',right_on='make_code',how='inner')\n",
    "X_test2=X_test2.rename(columns={\"subcat\":\"predicted_subcat_name\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>make_code</th>\n",
       "      <th>subcat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   make_code   subcat\n",
       "0        110  unknown"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xxxx=np.argmax(model.predict((w2v_model['lamb'].reshape(-1,100)+w2v_model['heart'].reshape(-1,100))/2),axis=1)\n",
    "subcat_index_mapping[subcat_index_mapping['make_code']==xxxx[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>final_subcat_name</th>\n",
       "      <th>predicted_subcat_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>boosey oma fav spiced gouda</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>kapiti gouda garlic chive</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>chrystal trio cheese selction</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>south cape cheese gouda block</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>biodynamic organic cheddar</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>warrnambool cheddar matured classic</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>south cape cheese sundried tomato</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>brancourt cheese colby</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>puhoi valley creamy blue</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>castello cheese blue slouse</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>bettum cheese savoury chive</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>meyer tasty gouda</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>ashgrove cheese golden valley</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>waimatum creamy blue vein</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>barry bay cheese cumin seed</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>kcf mango macadamium cheese</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>margaret river camembert</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>moondarra cream cheese bruschettum</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>president le blue creamy</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>cremeux dbl cream brie</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>warrnambool cheddar vintage classic</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>jindi cheese triple cream brie</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>waimatum creamy blue vein</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>aust gold crm cheeseonion chive</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>frico cheese wedge gouda light</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>warrnambool cheddar tomato chive</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>kcf mango macadamium cheese</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>capel valley cheese biscuit pck</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>south cape cheese twist ch</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>tasmanian heritage trpl cream brie</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93429</th>\n",
       "      <td>south cape fat cow milk fettum</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93430</th>\n",
       "      <td>president de bellay goat cheese</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93431</th>\n",
       "      <td>elco cheese fettum bulgarian</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93432</th>\n",
       "      <td>premium dairy cheese fettum fat</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93433</th>\n",
       "      <td>meredith marinated goat fettum</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93434</th>\n",
       "      <td>culinary fettum brine</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93435</th>\n",
       "      <td>pantalica ricottum fat cheese</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93436</th>\n",
       "      <td>hmbl plesure goat fetum oilw herb</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93437</th>\n",
       "      <td>cheese fettum aust fat</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93438</th>\n",
       "      <td>jarlsberg cheese lite shaved</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93439</th>\n",
       "      <td>dodoni cheese fettum greek</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93440</th>\n",
       "      <td>hillwood cheese fettum marinated</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93441</th>\n",
       "      <td>hillwood fetum danishstyle</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93442</th>\n",
       "      <td>hmbl plesure goat fetum oilw herb</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93443</th>\n",
       "      <td>meredith goat fettum</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93444</th>\n",
       "      <td>woodside goat curd</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93445</th>\n",
       "      <td>yarra valley persian fettum</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93446</th>\n",
       "      <td>pantalica ricottum cheese rw</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93447</th>\n",
       "      <td>yarra valley persian fettum</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93448</th>\n",
       "      <td>mamma lucium fat ricottum</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93449</th>\n",
       "      <td>premium dairy cheese fettum fat</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93450</th>\n",
       "      <td>pantalica ricottum fat cheese</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93451</th>\n",
       "      <td>hillwood cheese fettum marinated</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93452</th>\n",
       "      <td>la casa cheese ricottum fat</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93465</th>\n",
       "      <td>goat gouda</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93466</th>\n",
       "      <td>goat camembert</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93467</th>\n",
       "      <td>goat camembert</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93468</th>\n",
       "      <td>la buchette plain goat</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93469</th>\n",
       "      <td>yarra valley cheese dilly goat</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93470</th>\n",
       "      <td>south cape fettum sliced</td>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "      <td>CHEESE BULK</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>499 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   article               final_subcat_name  \\\n",
       "83             boosey oma fav spiced gouda  CHEESE PRE-PACKED ENTERTAINING   \n",
       "84               kapiti gouda garlic chive  CHEESE PRE-PACKED ENTERTAINING   \n",
       "85           chrystal trio cheese selction  CHEESE PRE-PACKED ENTERTAINING   \n",
       "86           south cape cheese gouda block  CHEESE PRE-PACKED ENTERTAINING   \n",
       "87              biodynamic organic cheddar  CHEESE PRE-PACKED ENTERTAINING   \n",
       "88     warrnambool cheddar matured classic  CHEESE PRE-PACKED ENTERTAINING   \n",
       "89       south cape cheese sundried tomato  CHEESE PRE-PACKED ENTERTAINING   \n",
       "90                  brancourt cheese colby  CHEESE PRE-PACKED ENTERTAINING   \n",
       "91                puhoi valley creamy blue  CHEESE PRE-PACKED ENTERTAINING   \n",
       "92             castello cheese blue slouse  CHEESE PRE-PACKED ENTERTAINING   \n",
       "93             bettum cheese savoury chive  CHEESE PRE-PACKED ENTERTAINING   \n",
       "94                       meyer tasty gouda  CHEESE PRE-PACKED ENTERTAINING   \n",
       "95           ashgrove cheese golden valley  CHEESE PRE-PACKED ENTERTAINING   \n",
       "96               waimatum creamy blue vein  CHEESE PRE-PACKED ENTERTAINING   \n",
       "97             barry bay cheese cumin seed  CHEESE PRE-PACKED ENTERTAINING   \n",
       "98             kcf mango macadamium cheese  CHEESE PRE-PACKED ENTERTAINING   \n",
       "99                margaret river camembert  CHEESE PRE-PACKED ENTERTAINING   \n",
       "100     moondarra cream cheese bruschettum  CHEESE PRE-PACKED ENTERTAINING   \n",
       "101               president le blue creamy  CHEESE PRE-PACKED ENTERTAINING   \n",
       "102                 cremeux dbl cream brie  CHEESE PRE-PACKED ENTERTAINING   \n",
       "103    warrnambool cheddar vintage classic  CHEESE PRE-PACKED ENTERTAINING   \n",
       "104         jindi cheese triple cream brie  CHEESE PRE-PACKED ENTERTAINING   \n",
       "105              waimatum creamy blue vein  CHEESE PRE-PACKED ENTERTAINING   \n",
       "106        aust gold crm cheeseonion chive  CHEESE PRE-PACKED ENTERTAINING   \n",
       "107         frico cheese wedge gouda light  CHEESE PRE-PACKED ENTERTAINING   \n",
       "108       warrnambool cheddar tomato chive  CHEESE PRE-PACKED ENTERTAINING   \n",
       "109            kcf mango macadamium cheese  CHEESE PRE-PACKED ENTERTAINING   \n",
       "110        capel valley cheese biscuit pck  CHEESE PRE-PACKED ENTERTAINING   \n",
       "111             south cape cheese twist ch  CHEESE PRE-PACKED ENTERTAINING   \n",
       "112     tasmanian heritage trpl cream brie  CHEESE PRE-PACKED ENTERTAINING   \n",
       "...                                    ...                             ...   \n",
       "93429       south cape fat cow milk fettum                     CHEESE BULK   \n",
       "93430      president de bellay goat cheese                     CHEESE BULK   \n",
       "93431         elco cheese fettum bulgarian                     CHEESE BULK   \n",
       "93432      premium dairy cheese fettum fat                     CHEESE BULK   \n",
       "93433       meredith marinated goat fettum                     CHEESE BULK   \n",
       "93434                culinary fettum brine                     CHEESE BULK   \n",
       "93435        pantalica ricottum fat cheese                     CHEESE BULK   \n",
       "93436    hmbl plesure goat fetum oilw herb                     CHEESE BULK   \n",
       "93437               cheese fettum aust fat                     CHEESE BULK   \n",
       "93438         jarlsberg cheese lite shaved                     CHEESE BULK   \n",
       "93439           dodoni cheese fettum greek                     CHEESE BULK   \n",
       "93440     hillwood cheese fettum marinated                     CHEESE BULK   \n",
       "93441           hillwood fetum danishstyle                     CHEESE BULK   \n",
       "93442    hmbl plesure goat fetum oilw herb                     CHEESE BULK   \n",
       "93443                 meredith goat fettum                     CHEESE BULK   \n",
       "93444                   woodside goat curd                     CHEESE BULK   \n",
       "93445          yarra valley persian fettum                     CHEESE BULK   \n",
       "93446         pantalica ricottum cheese rw                     CHEESE BULK   \n",
       "93447          yarra valley persian fettum                     CHEESE BULK   \n",
       "93448            mamma lucium fat ricottum                     CHEESE BULK   \n",
       "93449      premium dairy cheese fettum fat                     CHEESE BULK   \n",
       "93450        pantalica ricottum fat cheese                     CHEESE BULK   \n",
       "93451     hillwood cheese fettum marinated                     CHEESE BULK   \n",
       "93452          la casa cheese ricottum fat                     CHEESE BULK   \n",
       "93465                           goat gouda  CHEESE PRE-PACKED ENTERTAINING   \n",
       "93466                       goat camembert  CHEESE PRE-PACKED ENTERTAINING   \n",
       "93467                       goat camembert  CHEESE PRE-PACKED ENTERTAINING   \n",
       "93468               la buchette plain goat                  DAIRY - CHEESE   \n",
       "93469       yarra valley cheese dilly goat       CHEESE PRE-PACKED COOKING   \n",
       "93470             south cape fettum sliced       CHEESE PRE-PACKED COOKING   \n",
       "\n",
       "           predicted_subcat_name  \n",
       "83                DAIRY - CHEESE  \n",
       "84                DAIRY - CHEESE  \n",
       "85                DAIRY - CHEESE  \n",
       "86                DAIRY - CHEESE  \n",
       "87                DAIRY - CHEESE  \n",
       "88                DAIRY - CHEESE  \n",
       "89                DAIRY - CHEESE  \n",
       "90                DAIRY - CHEESE  \n",
       "91                DAIRY - CHEESE  \n",
       "92                DAIRY - CHEESE  \n",
       "93                DAIRY - CHEESE  \n",
       "94                DAIRY - CHEESE  \n",
       "95                DAIRY - CHEESE  \n",
       "96                DAIRY - CHEESE  \n",
       "97                DAIRY - CHEESE  \n",
       "98                DAIRY - CHEESE  \n",
       "99                DAIRY - CHEESE  \n",
       "100               DAIRY - CHEESE  \n",
       "101               DAIRY - CHEESE  \n",
       "102               DAIRY - CHEESE  \n",
       "103               DAIRY - CHEESE  \n",
       "104               DAIRY - CHEESE  \n",
       "105               DAIRY - CHEESE  \n",
       "106               DAIRY - CHEESE  \n",
       "107               DAIRY - CHEESE  \n",
       "108               DAIRY - CHEESE  \n",
       "109               DAIRY - CHEESE  \n",
       "110               DAIRY - CHEESE  \n",
       "111               DAIRY - CHEESE  \n",
       "112               DAIRY - CHEESE  \n",
       "...                          ...  \n",
       "93429  CHEESE PRE-PACKED COOKING  \n",
       "93430  CHEESE PRE-PACKED COOKING  \n",
       "93431  CHEESE PRE-PACKED COOKING  \n",
       "93432  CHEESE PRE-PACKED COOKING  \n",
       "93433  CHEESE PRE-PACKED COOKING  \n",
       "93434  CHEESE PRE-PACKED COOKING  \n",
       "93435  CHEESE PRE-PACKED COOKING  \n",
       "93436  CHEESE PRE-PACKED COOKING  \n",
       "93437  CHEESE PRE-PACKED COOKING  \n",
       "93438  CHEESE PRE-PACKED COOKING  \n",
       "93439  CHEESE PRE-PACKED COOKING  \n",
       "93440  CHEESE PRE-PACKED COOKING  \n",
       "93441  CHEESE PRE-PACKED COOKING  \n",
       "93442  CHEESE PRE-PACKED COOKING  \n",
       "93443  CHEESE PRE-PACKED COOKING  \n",
       "93444  CHEESE PRE-PACKED COOKING  \n",
       "93445  CHEESE PRE-PACKED COOKING  \n",
       "93446  CHEESE PRE-PACKED COOKING  \n",
       "93447  CHEESE PRE-PACKED COOKING  \n",
       "93448  CHEESE PRE-PACKED COOKING  \n",
       "93449  CHEESE PRE-PACKED COOKING  \n",
       "93450  CHEESE PRE-PACKED COOKING  \n",
       "93451  CHEESE PRE-PACKED COOKING  \n",
       "93452  CHEESE PRE-PACKED COOKING  \n",
       "93465                CHEESE BULK  \n",
       "93466                CHEESE BULK  \n",
       "93467                CHEESE BULK  \n",
       "93468                CHEESE BULK  \n",
       "93469                CHEESE BULK  \n",
       "93470                CHEESE BULK  \n",
       "\n",
       "[499 rows x 3 columns]"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test2[(X_test2['correct']==0) & (X_test2['final_subcat_name'].str.contains(\"CHEESE\")) & (X_test2['predicted_subcat_name'].str.contains(\"CHEESE\"))][['article','final_subcat_name','predicted_subcat_name']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y=pd.crosstab(X_test2['final_subcat_name'],X_test2['predicted_subcat_name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>predicted_subcat_name</th>\n",
       "      <th>ASIAN FOODS</th>\n",
       "      <th>BABY - BASICS</th>\n",
       "      <th>BABY - CARE</th>\n",
       "      <th>BABY - NAPPIES</th>\n",
       "      <th>BABY FOOD</th>\n",
       "      <th>BABY FORMULA</th>\n",
       "      <th>BEEF</th>\n",
       "      <th>BEEF CASE READY</th>\n",
       "      <th>BISCUITS - CRISPBREAD &amp; CRACKER</th>\n",
       "      <th>BISCUITS - PLAIN &amp; FANCY</th>\n",
       "      <th>...</th>\n",
       "      <th>SOFT DRINKS - WATER</th>\n",
       "      <th>SOUPS</th>\n",
       "      <th>SPREADS - HONEY</th>\n",
       "      <th>SPREADS - JAM</th>\n",
       "      <th>SPREADS - OTHER</th>\n",
       "      <th>SPREADS - PEANUT BUTTER</th>\n",
       "      <th>SUGAR</th>\n",
       "      <th>SUSHI</th>\n",
       "      <th>TEA</th>\n",
       "      <th>unknown</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>final_subcat_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ASIAN FOODS</th>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BABY - BASICS</th>\n",
       "      <td>0</td>\n",
       "      <td>210</td>\n",
       "      <td>17</td>\n",
       "      <td>10</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BABY - CARE</th>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>106</td>\n",
       "      <td>16</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BABY - NAPPIES</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>227</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BABY FOOD</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>432</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 99 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "predicted_subcat_name  ASIAN FOODS  BABY - BASICS  BABY - CARE  \\\n",
       "final_subcat_name                                                \n",
       "ASIAN FOODS                    172              0            0   \n",
       "BABY - BASICS                    0            210           17   \n",
       "BABY - CARE                      0              8          106   \n",
       "BABY - NAPPIES                   0              3            2   \n",
       "BABY FOOD                        1              3            3   \n",
       "\n",
       "predicted_subcat_name  BABY - NAPPIES  BABY FOOD  BABY FORMULA  BEEF  \\\n",
       "final_subcat_name                                                      \n",
       "ASIAN FOODS                         0          4             0     0   \n",
       "BABY - BASICS                      10         15             2     0   \n",
       "BABY - CARE                        16          4             0     0   \n",
       "BABY - NAPPIES                    227          0             0     0   \n",
       "BABY FOOD                           1        432             0     0   \n",
       "\n",
       "predicted_subcat_name  BEEF CASE READY  BISCUITS - CRISPBREAD & CRACKER  \\\n",
       "final_subcat_name                                                         \n",
       "ASIAN FOODS                          0                                1   \n",
       "BABY - BASICS                        0                                0   \n",
       "BABY - CARE                          0                                0   \n",
       "BABY - NAPPIES                       0                                0   \n",
       "BABY FOOD                            0                                0   \n",
       "\n",
       "predicted_subcat_name  BISCUITS - PLAIN & FANCY   ...     SOFT DRINKS - WATER  \\\n",
       "final_subcat_name                                 ...                           \n",
       "ASIAN FOODS                                   4   ...                       2   \n",
       "BABY - BASICS                                 0   ...                       2   \n",
       "BABY - CARE                                   0   ...                       0   \n",
       "BABY - NAPPIES                                0   ...                       0   \n",
       "BABY FOOD                                     0   ...                       0   \n",
       "\n",
       "predicted_subcat_name  SOUPS  SPREADS - HONEY  SPREADS - JAM  SPREADS - OTHER  \\\n",
       "final_subcat_name                                                               \n",
       "ASIAN FOODS                2                0              3                0   \n",
       "BABY - BASICS              0                0              0                0   \n",
       "BABY - CARE                0                0              0                0   \n",
       "BABY - NAPPIES             0                0              0                0   \n",
       "BABY FOOD                  0                1              0                0   \n",
       "\n",
       "predicted_subcat_name  SPREADS - PEANUT BUTTER  SUGAR  SUSHI  TEA  unknown  \n",
       "final_subcat_name                                                           \n",
       "ASIAN FOODS                                  0      3      8    5      488  \n",
       "BABY - BASICS                                0      0      0    0      806  \n",
       "BABY - CARE                                  0      0      0    0      134  \n",
       "BABY - NAPPIES                               0      0      0    0       48  \n",
       "BABY FOOD                                    0      0      0    0      171  \n",
       "\n",
       "[5 rows x 99 columns]"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y.to_csv(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/prediction_pivot14.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\gensim\\models\\phrases.py:274: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['didnt', 'enough', 'fetum', 'cheese', 'spread', 'light']"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comment=\"didnt have enough feta cheese spread lights\"\n",
    "bigram[preprocess(comment).split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['fetum', 'cheese']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\gensim\\models\\phrases.py:274: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    }
   ],
   "source": [
    "j=3\n",
    "x=bigram[(preprocess(comments.loc[j][0])).split()]\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "my_index=np.argmax(model.predict(w2v_model['john_west'].reshape(-1,100)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>make_code</th>\n",
       "      <th>subcat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14</td>\n",
       "      <td>CANNED FISH</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   make_code       subcat\n",
       "4         14  CANNED FISH"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subcat_index_mapping[subcat_index_mapping['make_code']==my_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mainland cheddar available</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cholesterol reducing cheese slice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>didnt enough kraft cheese spread light</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fetum cheese</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>toffutti cream cheese deleted</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  Comment\n",
       "0              mainland cheddar available\n",
       "1       cholesterol reducing cheese slice\n",
       "2  didnt enough kraft cheese spread light\n",
       "3                            fetum cheese\n",
       "4           toffutti cream cheese deleted"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\gensim\\models\\phrases.py:274: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n",
      "100\n",
      "100\n",
      "200\n",
      "200\n",
      "300\n",
      "300\n",
      "400\n",
      "400\n",
      "500\n",
      "500\n",
      "600\n",
      "600\n",
      "700\n",
      "700\n",
      "800\n",
      "800\n",
      "900\n",
      "900\n",
      "1000\n",
      "1000\n",
      "1100\n",
      "1100\n",
      "1200\n",
      "1200\n",
      "1300\n",
      "1300\n",
      "1400\n",
      "1400\n",
      "1500\n",
      "1500\n",
      "1600\n",
      "1600\n",
      "1700\n",
      "1700\n",
      "1800\n",
      "1800\n",
      "1900\n",
      "1900\n",
      "2000\n",
      "2000\n",
      "2100\n",
      "2100\n",
      "2200\n",
      "2200\n",
      "2300\n",
      "2300\n",
      "2400\n",
      "2400\n",
      "2500\n",
      "2500\n",
      "2600\n",
      "2600\n",
      "2700\n",
      "2700\n",
      "2800\n",
      "2800\n",
      "2900\n",
      "2900\n",
      "3000\n",
      "3000\n",
      "3100\n",
      "3100\n",
      "3200\n",
      "3200\n",
      "3300\n",
      "3300\n",
      "3400\n",
      "3400\n",
      "3500\n",
      "3500\n",
      "3600\n",
      "3600\n",
      "3700\n",
      "3700\n",
      "3800\n",
      "3800\n",
      "3900\n",
      "3900\n",
      "4000\n",
      "4000\n",
      "4100\n",
      "4100\n",
      "4200\n",
      "4200\n",
      "4300\n",
      "4300\n",
      "4400\n",
      "4400\n",
      "4500\n",
      "4500\n",
      "4600\n",
      "4600\n",
      "4700\n",
      "4700\n",
      "4800\n",
      "4800\n",
      "4900\n",
      "4900\n",
      "5000\n",
      "5000\n",
      "5100\n",
      "5100\n",
      "5200\n",
      "5200\n",
      "5300\n",
      "5300\n",
      "5400\n",
      "5400\n",
      "5500\n",
      "5500\n",
      "5600\n",
      "5600\n",
      "5700\n",
      "5700\n",
      "5800\n",
      "5800\n",
      "5900\n",
      "5900\n",
      "6000\n",
      "6000\n",
      "6100\n",
      "6100\n",
      "6200\n",
      "6200\n",
      "6300\n",
      "6300\n",
      "6400\n",
      "6400\n",
      "6500\n",
      "6500\n",
      "6600\n",
      "6600\n",
      "6700\n",
      "6700\n",
      "6800\n",
      "6800\n",
      "6900\n",
      "6900\n",
      "7000\n",
      "7000\n",
      "7100\n",
      "7100\n",
      "7200\n",
      "7200\n",
      "7300\n",
      "7300\n",
      "7400\n",
      "7400\n",
      "7500\n",
      "7500\n",
      "7600\n",
      "7600\n",
      "7700\n",
      "7700\n",
      "7800\n",
      "7800\n",
      "7900\n",
      "7900\n",
      "8000\n",
      "8000\n",
      "8100\n",
      "8100\n",
      "8200\n",
      "8200\n",
      "8300\n",
      "8300\n",
      "8400\n",
      "8400\n",
      "8500\n",
      "8500\n",
      "8600\n",
      "8600\n",
      "8700\n",
      "8700\n",
      "8800\n",
      "8800\n",
      "8900\n",
      "8900\n",
      "9000\n",
      "9000\n",
      "9100\n",
      "9100\n",
      "9200\n",
      "9200\n",
      "9300\n",
      "9300\n",
      "9400\n",
      "9400\n",
      "9500\n",
      "9500\n",
      "9600\n",
      "9600\n",
      "9700\n",
      "9700\n",
      "9800\n",
      "9800\n",
      "9900\n",
      "9900\n",
      "10000\n",
      "10000\n",
      "10100\n",
      "10100\n",
      "10200\n",
      "10200\n",
      "10300\n",
      "10300\n",
      "10400\n",
      "10400\n",
      "10500\n",
      "10500\n",
      "10600\n",
      "10600\n",
      "10700\n",
      "10700\n",
      "10800\n",
      "10800\n",
      "10900\n",
      "10900\n",
      "11000\n",
      "11000\n",
      "11100\n",
      "11100\n",
      "11200\n",
      "11200\n",
      "11300\n",
      "11300\n",
      "11400\n",
      "11400\n",
      "11500\n",
      "11500\n",
      "11600\n",
      "11600\n",
      "11700\n",
      "11700\n",
      "11800\n",
      "11800\n",
      "11900\n",
      "11900\n",
      "12000\n",
      "12000\n",
      "12100\n",
      "12100\n",
      "12200\n",
      "12200\n",
      "12300\n",
      "12300\n",
      "12400\n",
      "12400\n",
      "12500\n",
      "12500\n",
      "12600\n",
      "12600\n",
      "12700\n",
      "12700\n",
      "12800\n",
      "12800\n",
      "12900\n",
      "12900\n",
      "13000\n",
      "13000\n",
      "13100\n",
      "13100\n",
      "13200\n",
      "13200\n",
      "13300\n",
      "13300\n",
      "13400\n",
      "13400\n",
      "13500\n",
      "13500\n",
      "13600\n",
      "13600\n",
      "13700\n",
      "13700\n",
      "13800\n",
      "13800\n",
      "13900\n",
      "13900\n",
      "14000\n",
      "14000\n",
      "14100\n",
      "14100\n",
      "14200\n",
      "14200\n",
      "14300\n",
      "14300\n",
      "14400\n",
      "14400\n",
      "14500\n",
      "14500\n",
      "14600\n",
      "14600\n",
      "14700\n",
      "14700\n",
      "14800\n",
      "14800\n",
      "14900\n",
      "14900\n",
      "15000\n",
      "15000\n",
      "15100\n",
      "15100\n",
      "15200\n",
      "15200\n",
      "15300\n",
      "15300\n",
      "15400\n",
      "15400\n",
      "15500\n",
      "15500\n",
      "15600\n",
      "15600\n",
      "15700\n",
      "15700\n",
      "15800\n",
      "15800\n",
      "15900\n",
      "15900\n",
      "16000\n",
      "16000\n",
      "16100\n",
      "16100\n",
      "16200\n",
      "16200\n",
      "16300\n",
      "16300\n",
      "16400\n",
      "16400\n",
      "16500\n",
      "16500\n",
      "16600\n",
      "16600\n",
      "16700\n",
      "16700\n",
      "16800\n",
      "16800\n",
      "16900\n",
      "16900\n",
      "17000\n",
      "17000\n",
      "17100\n",
      "17100\n",
      "17200\n",
      "17200\n",
      "17300\n",
      "17300\n",
      "17400\n",
      "17400\n",
      "17500\n",
      "17500\n",
      "17600\n",
      "17600\n",
      "17700\n",
      "17700\n",
      "17800\n",
      "17800\n",
      "17900\n",
      "17900\n",
      "18000\n",
      "18000\n",
      "18100\n",
      "18100\n",
      "18200\n",
      "18200\n",
      "18300\n",
      "18300\n",
      "18400\n",
      "18400\n",
      "18500\n",
      "18500\n",
      "18600\n",
      "18600\n",
      "18700\n",
      "18700\n",
      "18800\n",
      "18800\n",
      "18900\n",
      "18900\n",
      "19000\n",
      "19000\n",
      "19100\n",
      "19100\n",
      "19200\n",
      "19200\n",
      "19300\n",
      "19300\n",
      "19400\n",
      "19400\n",
      "19500\n",
      "19500\n",
      "19600\n",
      "19600\n",
      "19700\n",
      "19700\n",
      "19800\n",
      "19800\n",
      "19900\n",
      "19900\n",
      "20000\n",
      "20000\n",
      "20100\n",
      "20100\n",
      "20200\n",
      "20200\n",
      "20300\n",
      "20300\n",
      "20400\n",
      "20400\n",
      "20500\n",
      "20500\n",
      "20600\n",
      "20600\n",
      "20700\n",
      "20700\n",
      "20800\n",
      "20800\n",
      "20900\n",
      "20900\n",
      "21000\n",
      "21000\n",
      "21100\n",
      "21100\n",
      "21200\n",
      "21200\n",
      "21300\n",
      "21300\n",
      "21400\n",
      "21400\n",
      "21500\n",
      "21500\n",
      "21600\n",
      "21600\n",
      "21700\n",
      "21700\n",
      "21800\n",
      "21800\n",
      "21900\n",
      "21900\n",
      "22000\n",
      "22000\n",
      "22100\n",
      "22100\n",
      "22200\n",
      "22200\n",
      "22300\n",
      "22300\n",
      "22400\n",
      "22400\n",
      "22500\n",
      "22500\n",
      "22600\n",
      "22600\n",
      "22700\n",
      "22700\n",
      "22800\n",
      "22800\n",
      "22900\n",
      "22900\n",
      "23000\n",
      "23000\n",
      "23100\n",
      "23100\n",
      "23200\n",
      "23200\n",
      "23300\n",
      "23300\n",
      "23400\n",
      "23400\n",
      "23500\n",
      "23500\n",
      "23600\n",
      "23600\n",
      "23700\n",
      "23700\n",
      "23800\n",
      "23800\n",
      "23900\n",
      "23900\n",
      "24000\n",
      "24000\n",
      "24100\n",
      "24100\n",
      "24200\n",
      "24200\n",
      "24300\n",
      "24300\n",
      "24400\n",
      "24400\n",
      "24500\n",
      "24500\n",
      "24600\n",
      "24600\n",
      "24700\n",
      "24700\n",
      "24800\n",
      "24800\n",
      "24900\n",
      "24900\n",
      "25000\n",
      "25000\n",
      "25100\n",
      "25100\n",
      "25200\n",
      "25200\n",
      "25300\n",
      "25300\n",
      "25400\n",
      "25400\n",
      "25500\n",
      "25500\n",
      "25600\n",
      "25600\n",
      "25700\n",
      "25700\n",
      "25800\n",
      "25800\n",
      "25900\n",
      "25900\n",
      "26000\n",
      "26000\n",
      "26100\n",
      "26100\n",
      "26200\n",
      "26200\n",
      "26300\n",
      "26300\n",
      "26400\n",
      "26400\n",
      "26500\n",
      "26500\n",
      "26600\n",
      "26600\n",
      "26700\n",
      "26700\n",
      "26800\n",
      "26800\n",
      "26900\n",
      "26900\n",
      "27000\n",
      "27000\n",
      "27100\n",
      "27100\n",
      "27200\n",
      "27200\n",
      "27300\n",
      "27300\n",
      "27400\n",
      "27400\n",
      "27500\n",
      "27500\n",
      "27600\n",
      "27600\n",
      "27700\n",
      "27700\n",
      "27800\n",
      "27800\n",
      "27900\n",
      "27900\n",
      "28000\n",
      "28000\n",
      "28100\n",
      "28100\n",
      "28200\n",
      "28200\n",
      "28300\n",
      "28300\n",
      "28400\n",
      "28400\n",
      "28500\n",
      "28500\n",
      "28600\n",
      "28600\n",
      "28700\n",
      "28700\n",
      "28800\n",
      "28800\n",
      "28900\n",
      "28900\n",
      "29000\n",
      "29000\n",
      "29100\n",
      "29100\n",
      "29200\n",
      "29200\n",
      "29300\n",
      "29300\n",
      "29400\n",
      "29400\n",
      "29500\n",
      "29500\n",
      "29600\n",
      "29600\n",
      "29700\n",
      "29700\n",
      "29800\n",
      "29800\n",
      "29900\n",
      "29900\n",
      "30000\n",
      "30000\n",
      "30100\n",
      "30100\n",
      "30200\n",
      "30200\n",
      "30300\n",
      "30300\n",
      "30400\n",
      "30400\n",
      "30500\n",
      "30500\n",
      "30600\n",
      "30600\n",
      "30700\n",
      "30700\n",
      "30800\n",
      "30800\n",
      "30900\n",
      "30900\n",
      "31000\n",
      "31000\n",
      "31100\n",
      "31100\n",
      "31200\n",
      "31200\n",
      "31300\n",
      "31300\n",
      "31400\n",
      "31400\n",
      "31500\n",
      "31500\n",
      "31600\n",
      "31600\n",
      "31700\n",
      "31700\n",
      "31800\n",
      "31800\n",
      "31900\n",
      "31900\n",
      "32000\n",
      "32000\n",
      "32100\n",
      "32100\n",
      "32200\n",
      "32200\n",
      "32300\n",
      "32300\n",
      "32400\n",
      "32400\n",
      "32500\n",
      "32500\n",
      "32600\n",
      "32600\n",
      "32700\n",
      "32700\n",
      "32800\n",
      "32800\n",
      "32900\n",
      "32900\n",
      "33000\n",
      "33000\n",
      "33100\n",
      "33100\n",
      "33200\n",
      "33200\n",
      "33300\n",
      "33300\n",
      "33400\n",
      "33400\n",
      "33500\n",
      "33500\n",
      "33600\n",
      "33600\n",
      "33700\n",
      "33700\n",
      "33800\n",
      "33800\n",
      "33900\n",
      "33900\n",
      "34000\n",
      "34000\n",
      "34100\n",
      "34100\n",
      "34200\n",
      "34200\n",
      "34300\n",
      "34300\n",
      "34400\n",
      "34400\n",
      "34500\n",
      "34500\n",
      "34600\n",
      "34600\n",
      "34700\n",
      "34700\n",
      "34800\n",
      "34800\n",
      "34900\n",
      "34900\n",
      "35000\n",
      "35000\n",
      "35100\n",
      "35100\n",
      "35200\n",
      "35200\n",
      "35300\n",
      "35300\n",
      "35400\n",
      "35400\n",
      "35500\n",
      "35500\n",
      "35600\n",
      "35600\n",
      "35700\n",
      "35700\n",
      "35800\n",
      "35800\n",
      "35900\n",
      "35900\n",
      "36000\n",
      "36000\n",
      "36100\n",
      "36100\n",
      "36200\n",
      "36200\n",
      "36300\n",
      "36300\n",
      "36400\n",
      "36400\n",
      "36500\n",
      "36500\n",
      "36600\n",
      "36600\n",
      "36700\n",
      "36700\n",
      "36800\n",
      "36800\n",
      "36900\n",
      "36900\n",
      "37000\n",
      "37000\n",
      "37100\n",
      "37100\n",
      "37200\n",
      "37200\n",
      "37300\n",
      "37300\n",
      "37400\n",
      "37400\n",
      "37500\n",
      "37500\n",
      "37600\n",
      "37600\n",
      "37700\n",
      "37700\n",
      "37800\n",
      "37800\n",
      "37900\n",
      "37900\n",
      "38000\n",
      "38000\n",
      "38100\n",
      "38100\n",
      "38200\n",
      "38200\n",
      "38300\n",
      "38300\n",
      "38400\n",
      "38400\n",
      "38500\n",
      "38500\n",
      "38600\n",
      "38600\n",
      "38700\n",
      "38700\n",
      "38800\n",
      "38800\n",
      "38900\n",
      "38900\n",
      "39000\n",
      "39000\n",
      "39100\n",
      "39100\n",
      "39200\n",
      "39200\n",
      "39300\n",
      "39300\n",
      "39400\n",
      "39400\n",
      "39500\n",
      "39500\n",
      "39600\n",
      "39600\n",
      "39700\n",
      "39700\n",
      "39800\n",
      "39800\n",
      "39900\n",
      "39900\n",
      "40000\n",
      "40000\n",
      "40100\n",
      "40100\n",
      "40200\n",
      "40200\n",
      "40300\n",
      "40300\n",
      "40400\n",
      "40400\n",
      "40500\n",
      "40500\n",
      "40600\n",
      "40600\n",
      "40700\n",
      "40700\n",
      "40800\n",
      "40800\n",
      "40900\n",
      "40900\n",
      "41000\n",
      "41000\n",
      "41100\n",
      "41100\n",
      "41200\n",
      "41200\n",
      "41300\n",
      "41300\n",
      "41400\n",
      "41400\n",
      "41500\n",
      "41500\n",
      "41600\n",
      "41600\n",
      "41700\n",
      "41700\n",
      "41800\n",
      "41800\n",
      "41900\n",
      "41900\n",
      "42000\n",
      "42000\n",
      "42100\n",
      "42100\n",
      "42200\n",
      "42200\n",
      "42300\n",
      "42300\n",
      "42400\n",
      "42400\n",
      "42500\n",
      "42500\n",
      "42600\n",
      "42600\n",
      "42700\n",
      "42700\n",
      "42800\n",
      "42800\n",
      "42900\n",
      "42900\n",
      "43000\n",
      "43000\n",
      "43100\n",
      "43100\n",
      "43200\n",
      "43200\n",
      "43300\n",
      "43300\n",
      "43400\n",
      "43400\n",
      "43500\n",
      "43500\n",
      "43600\n",
      "43600\n",
      "43700\n",
      "43700\n",
      "43800\n",
      "43800\n",
      "43900\n",
      "43900\n",
      "44000\n",
      "44000\n",
      "44100\n",
      "44100\n",
      "44200\n",
      "44200\n",
      "44300\n",
      "44300\n",
      "44400\n",
      "44400\n",
      "44500\n",
      "44500\n",
      "44600\n",
      "44600\n",
      "44700\n",
      "44700\n",
      "44800\n",
      "44800\n",
      "44900\n",
      "44900\n",
      "45000\n",
      "45000\n",
      "45100\n",
      "45100\n",
      "45200\n",
      "45200\n",
      "45300\n",
      "45300\n",
      "45400\n",
      "45400\n",
      "45500\n",
      "45500\n",
      "45600\n",
      "45600\n",
      "45700\n",
      "45700\n",
      "45800\n",
      "45800\n",
      "45900\n",
      "45900\n",
      "46000\n",
      "46000\n",
      "46100\n",
      "46100\n",
      "46200\n",
      "46200\n",
      "46300\n",
      "46300\n",
      "46400\n",
      "46400\n",
      "46500\n",
      "46500\n",
      "46600\n",
      "46600\n",
      "46700\n",
      "46700\n",
      "46800\n",
      "46800\n",
      "46900\n",
      "46900\n",
      "47000\n",
      "47000\n",
      "47100\n",
      "47100\n",
      "47200\n",
      "47200\n",
      "47300\n",
      "47300\n",
      "47400\n",
      "47400\n",
      "47500\n",
      "47500\n",
      "47600\n",
      "47600\n",
      "47700\n",
      "47700\n",
      "47800\n",
      "47800\n",
      "47900\n",
      "47900\n",
      "48000\n",
      "48000\n",
      "48100\n",
      "48100\n",
      "48200\n",
      "48200\n",
      "48300\n",
      "48300\n",
      "48400\n",
      "48400\n",
      "48500\n",
      "48500\n",
      "48600\n",
      "48600\n",
      "48700\n",
      "48700\n",
      "48800\n",
      "48800\n",
      "48900\n",
      "48900\n",
      "49000\n",
      "49000\n",
      "49100\n",
      "49100\n",
      "49200\n",
      "49200\n",
      "49300\n",
      "49300\n",
      "49400\n",
      "49400\n",
      "49500\n",
      "49500\n",
      "49600\n",
      "49600\n",
      "49700\n",
      "49700\n",
      "49800\n",
      "49800\n",
      "49900\n",
      "49900\n",
      "50000\n",
      "50000\n",
      "50100\n",
      "50100\n",
      "50200\n",
      "50200\n",
      "50300\n",
      "50300\n",
      "50400\n",
      "50400\n",
      "50500\n",
      "50500\n",
      "50600\n",
      "50600\n",
      "50700\n",
      "50700\n",
      "50800\n",
      "50800\n",
      "50900\n",
      "50900\n",
      "51000\n",
      "51000\n",
      "51100\n",
      "51100\n",
      "51200\n",
      "51200\n",
      "51300\n",
      "51300\n",
      "51400\n",
      "51400\n",
      "51500\n",
      "51500\n",
      "51600\n",
      "51600\n",
      "51700\n",
      "51700\n",
      "51800\n",
      "51800\n",
      "51900\n",
      "51900\n",
      "52000\n",
      "52000\n",
      "52100\n",
      "52100\n",
      "52200\n",
      "52200\n",
      "52300\n",
      "52300\n",
      "52400\n",
      "52400\n",
      "52500\n",
      "52500\n",
      "52600\n",
      "52600\n",
      "52700\n",
      "52700\n",
      "52800\n",
      "52800\n",
      "52900\n",
      "52900\n",
      "53000\n",
      "53000\n",
      "53100\n",
      "53100\n",
      "53200\n",
      "53200\n",
      "53300\n",
      "53300\n",
      "53400\n",
      "53400\n",
      "53500\n",
      "53500\n",
      "53600\n",
      "53600\n",
      "53700\n",
      "53700\n",
      "53800\n",
      "53800\n",
      "53900\n",
      "53900\n",
      "54000\n",
      "54000\n",
      "54100\n",
      "54100\n",
      "54200\n",
      "54200\n",
      "54300\n",
      "54300\n",
      "54400\n",
      "54400\n",
      "54500\n",
      "54500\n",
      "54600\n",
      "54600\n",
      "54700\n",
      "54700\n",
      "54800\n",
      "54800\n",
      "54900\n",
      "54900\n",
      "55000\n",
      "55000\n",
      "55100\n",
      "55100\n",
      "55200\n",
      "55200\n",
      "55300\n",
      "55300\n",
      "55400\n",
      "55400\n",
      "55500\n",
      "55500\n",
      "55600\n",
      "55600\n",
      "55700\n",
      "55700\n",
      "55800\n",
      "55800\n",
      "55900\n",
      "55900\n",
      "56000\n",
      "56000\n",
      "56100\n",
      "56100\n",
      "56200\n",
      "56200\n",
      "56300\n",
      "56300\n",
      "56400\n",
      "56400\n",
      "56500\n",
      "56500\n",
      "56600\n",
      "56600\n",
      "56700\n",
      "56700\n",
      "56800\n",
      "56800\n",
      "56900\n",
      "56900\n",
      "57000\n",
      "57000\n",
      "57100\n",
      "57100\n",
      "57200\n",
      "57200\n",
      "57300\n",
      "57300\n",
      "57400\n",
      "57400\n",
      "57500\n",
      "57500\n",
      "57600\n",
      "57600\n",
      "57700\n",
      "57700\n",
      "57800\n",
      "57800\n",
      "57900\n",
      "57900\n",
      "58000\n",
      "58000\n",
      "58100\n",
      "58100\n",
      "58200\n",
      "58200\n",
      "58300\n",
      "58300\n",
      "58400\n",
      "58400\n",
      "58500\n",
      "58500\n",
      "58600\n",
      "58600\n",
      "58700\n",
      "58700\n",
      "58800\n",
      "58800\n",
      "58900\n",
      "58900\n",
      "59000\n",
      "59000\n",
      "59100\n",
      "59100\n",
      "59200\n",
      "59200\n",
      "59300\n",
      "59300\n",
      "59400\n",
      "59400\n",
      "59500\n",
      "59500\n",
      "59600\n",
      "59600\n",
      "59700\n",
      "59700\n",
      "59800\n",
      "59800\n",
      "59900\n",
      "59900\n",
      "60000\n",
      "60000\n",
      "60100\n",
      "60100\n",
      "60200\n",
      "60200\n",
      "60300\n",
      "60300\n",
      "60400\n",
      "60400\n",
      "60500\n",
      "60500\n",
      "60600\n",
      "60600\n",
      "60700\n",
      "60700\n",
      "60800\n",
      "60800\n",
      "60900\n",
      "60900\n",
      "61000\n",
      "61000\n",
      "61100\n",
      "61100\n",
      "61200\n",
      "61200\n",
      "61300\n",
      "61300\n",
      "61400\n",
      "61400\n",
      "61500\n",
      "61500\n",
      "61600\n",
      "61600\n",
      "61700\n",
      "61700\n",
      "61800\n",
      "61800\n",
      "61900\n",
      "61900\n",
      "62000\n",
      "62000\n",
      "62100\n",
      "62100\n",
      "62200\n",
      "62200\n",
      "62300\n",
      "62300\n",
      "62400\n",
      "62400\n",
      "62500\n",
      "62500\n",
      "62600\n",
      "62600\n",
      "62700\n",
      "62700\n",
      "62800\n",
      "62800\n",
      "62900\n",
      "62900\n",
      "63000\n",
      "63000\n",
      "63100\n",
      "63100\n",
      "63200\n",
      "63200\n",
      "63300\n",
      "63300\n",
      "63400\n",
      "63400\n",
      "63500\n",
      "63500\n",
      "63600\n",
      "63600\n",
      "63700\n",
      "63700\n",
      "63800\n",
      "63800\n",
      "63900\n",
      "63900\n",
      "64000\n",
      "64000\n",
      "64100\n",
      "64100\n",
      "64200\n",
      "64200\n",
      "64300\n",
      "64300\n",
      "64400\n",
      "64400\n",
      "64500\n",
      "64500\n",
      "64600\n",
      "64600\n",
      "64700\n",
      "64700\n",
      "64800\n",
      "64800\n",
      "64900\n",
      "64900\n",
      "65000\n",
      "65000\n",
      "65100\n",
      "65100\n",
      "65200\n",
      "65200\n",
      "65300\n",
      "65300\n",
      "65400\n",
      "65400\n",
      "65500\n",
      "65500\n",
      "65600\n",
      "65600\n",
      "65700\n",
      "65700\n",
      "65800\n",
      "65800\n",
      "65900\n",
      "65900\n",
      "66000\n",
      "66000\n",
      "66100\n",
      "66100\n",
      "66200\n",
      "66200\n",
      "66300\n",
      "66300\n",
      "66400\n",
      "66400\n",
      "66500\n",
      "66500\n",
      "66600\n",
      "66600\n",
      "66700\n",
      "66700\n",
      "66800\n",
      "66800\n",
      "66900\n",
      "66900\n",
      "67000\n",
      "67000\n",
      "67100\n",
      "67100\n",
      "67200\n",
      "67200\n",
      "67300\n",
      "67300\n",
      "67400\n",
      "67400\n",
      "67500\n",
      "67500\n",
      "67600\n",
      "67600\n",
      "67700\n",
      "67700\n",
      "67800\n",
      "67800\n",
      "67900\n",
      "67900\n",
      "68000\n",
      "68000\n",
      "68100\n",
      "68100\n",
      "68200\n",
      "68200\n",
      "68300\n",
      "68300\n",
      "68400\n",
      "68400\n",
      "68500\n",
      "68500\n",
      "68600\n",
      "68600\n",
      "68700\n",
      "68700\n",
      "68800\n",
      "68800\n",
      "68900\n",
      "68900\n",
      "69000\n",
      "69000\n"
     ]
    }
   ],
   "source": [
    "# pred=[]\n",
    "# pred_score=[]\n",
    "# for j in range(comments.shape[0]):\n",
    "#     x=bigram[(preprocess(comments.loc[j][0])).split()]\n",
    "#     y=[0,0]\n",
    "#     k=0\n",
    "#     m=0\n",
    "#     z2=0\n",
    "#     if(j%100==0):\n",
    "#         print(j)\n",
    "#         print(len(pred))\n",
    "#     if(len(x)>0):\n",
    "        \n",
    "#         for i in range(len(x)):\n",
    "#             try:\n",
    "#                 z2=z2+w2v_model[x[i]].reshape(-1,100)\n",
    "#                 #y=y+model2.predict_proba(model[x[i]].reshape(-1,100))\n",
    "#                 m=m+1\n",
    "#             except KeyError:\n",
    "#                 k=k+1\n",
    "    \n",
    "#         if(m>0):\n",
    "#             z=model.predict(z2/m)\n",
    "#         else:\n",
    "#             z=[0]\n",
    "#         #print(comments.loc[j][0])\n",
    "#     else:\n",
    "#         z=[0]\n",
    "#     try:\n",
    "        \n",
    "\n",
    "#         pred.append(subcat_index_mapping[subcat_index_mapping['make_code']==np.argmax(z,axis=1)[0]]['subcat'].item())\n",
    "#         #print(subcat_index_mapping[subcat_index_mapping['make_code']==np.argmax(z,axis=1)[0]]['subcat'].item())\n",
    "#         pred_score.append(z[0][np.argmax(z,axis=1)][0])\n",
    "       \n",
    "#     except:\n",
    "#         pred.append(\"unknown2\")\n",
    "#         pred_score.append(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 977,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69045"
      ]
     },
     "execution_count": 977,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# len(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 978,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.473746, 0.29134691, 0.48425904, 0.40222928, 0.34226823]"
      ]
     },
     "execution_count": 978,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pred_score[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 979,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DAIRY - CHEESE',\n",
       " 'DAIRY - CHEESE',\n",
       " 'DAIRY - BUTTER & MARGARINE',\n",
       " 'CHEESE PRE-PACKED COOKING',\n",
       " 'DAIRY - CHEESE']"
      ]
     },
     "execution_count": 979,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pred[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "comments=pd.read_table(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/comments.csv\",encoding='latin1')\n",
    "# comments['predicted_subcat']=pred\n",
    "# comments['predicted_subcat_score']=pred_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'DAIRY - MILK'"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z=model.predict(w2v_model['milk'].reshape(-1,100))\n",
    "subcat_index_mapping[subcat_index_mapping['make_code']==np.argmax(z,axis=1)[0]]['subcat'].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mainland cheddar not available</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>no cholesterol reducing cheese slices</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>didnt have enough Kraft cheese spread light</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>No Feta cheese.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Toffutti cream cheese has been deleted</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Comment\n",
       "0               Mainland cheddar not available\n",
       "1        no cholesterol reducing cheese slices\n",
       "2  didnt have enough Kraft cheese spread light\n",
       "3                              No Feta cheese.\n",
       "4       Toffutti cream cheese has been deleted"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\gensim\\models\\phrases.py:274: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    }
   ],
   "source": [
    "zz=\"Fruits and vegetables I was looking for were not there..possibly run out\"\n",
    "xx=bigram[preprocess(zz).split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pred_subcat2=[]\n",
    "# for j in range(comments.shape[0]):\n",
    "#     xx=bigram[(preprocess(comments.loc[j][0])).split()]\n",
    "#     y=(\"fruit\" in xx) + (\"veg\" in xx) + (\"vegetable\" in xx) \n",
    "#     z=(\"banana\" in xx)\n",
    "#     bbq=(\"bbq\" in xx) + (\"chicken\" in xx)\n",
    "#     meat = (\"meat\" in xx)\n",
    "#     if(j%100==0):\n",
    "#         print(j)\n",
    "#     if (y>0):\n",
    "#         pred_subcat2.append('CANNED VEGETABLES')\n",
    "#     elif (z==1):    \n",
    "#         pred_subcat2.append('PRODUCE')\n",
    "#     elif(bbq==2):\n",
    "#         pred_subcat2.append('SERVICED MEAT')\n",
    "#     elif (meat==1):\n",
    "#         pred_subcat2.append('SERVICED MEAT')\n",
    "#     else:\n",
    "#         pred_subcat2.append(comments.loc[j]['predicted_subcat'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pred_subcat2[280]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 986,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# comments=comments.drop(['predicted_subcat'],axis=1)\n",
    "# comments['predicted_subcat']=pred_subcat2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#comments.to_csv(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/comments_predicted_subcats22.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk import tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "comments=pd.read_table(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/comments.csv\",encoding='latin1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\gensim\\models\\phrases.py:274: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "100\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "300\n",
      "300\n",
      "300\n",
      "300\n",
      "300\n",
      "400\n",
      "400\n",
      "400\n",
      "400\n",
      "400\n",
      "400\n",
      "400\n",
      "400\n",
      "400\n",
      "500\n",
      "500\n",
      "600\n",
      "600\n",
      "600\n",
      "600\n",
      "700\n",
      "700\n",
      "700\n",
      "700\n",
      "700\n",
      "700\n",
      "700\n",
      "700\n",
      "700\n",
      "700\n",
      "700\n",
      "700\n",
      "800\n",
      "800\n",
      "800\n",
      "900\n",
      "900\n",
      "900\n",
      "1000\n",
      "1200\n",
      "1300\n",
      "1300\n",
      "1300\n",
      "1400\n",
      "1400\n",
      "1400\n",
      "1400\n",
      "1500\n",
      "1500\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1700\n",
      "1700\n",
      "1700\n",
      "1800\n",
      "1800\n",
      "1800\n",
      "1800\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2000\n",
      "2100\n",
      "2100\n",
      "2300\n",
      "2300\n",
      "2300\n",
      "2300\n",
      "2400\n",
      "2400\n",
      "2500\n",
      "2500\n",
      "2500\n",
      "2500\n",
      "2500\n",
      "2600\n",
      "2600\n",
      "2600\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2800\n",
      "2800\n",
      "2900\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3200\n",
      "3300\n",
      "3300\n",
      "3400\n",
      "3400\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4100\n",
      "4200\n",
      "4200\n",
      "4300\n",
      "4300\n",
      "4300\n",
      "4400\n",
      "4400\n",
      "4400\n",
      "4400\n",
      "4400\n",
      "4500\n",
      "4500\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4700\n",
      "4700\n",
      "4800\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5400\n",
      "5400\n",
      "5500\n",
      "5500\n",
      "5600\n",
      "5600\n",
      "5700\n",
      "5700\n",
      "5800\n",
      "5800\n",
      "5800\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6000\n",
      "6000\n",
      "6000\n",
      "6000\n",
      "6000\n",
      "6100\n",
      "6100\n",
      "6100\n",
      "6200\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6400\n",
      "6400\n",
      "6500\n",
      "6500\n",
      "6500\n",
      "6600\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6800\n",
      "6800\n",
      "6800\n",
      "6900\n",
      "6900\n",
      "7000\n",
      "7000\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7400\n",
      "7400\n",
      "7400\n",
      "7400\n",
      "7400\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7800\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8300\n",
      "8500\n",
      "8500\n",
      "8500\n",
      "8600\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "8900\n",
      "8900\n",
      "8900\n",
      "8900\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9100\n",
      "9200\n",
      "9200\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9700\n",
      "9800\n",
      "9800\n",
      "9800\n",
      "9900\n",
      "10000\n",
      "10000\n",
      "10200\n",
      "10200\n",
      "10300\n",
      "10300\n",
      "10300\n",
      "10300\n",
      "10400\n",
      "10500\n",
      "10500\n",
      "10600\n",
      "10600\n",
      "10700\n",
      "10700\n",
      "10800\n",
      "10800\n",
      "10900\n",
      "11000\n",
      "11100\n",
      "11200\n",
      "11300\n",
      "11300\n",
      "11300\n",
      "11400\n",
      "11400\n",
      "11400\n",
      "11500\n",
      "11600\n",
      "11700\n",
      "11900\n",
      "11900\n",
      "11900\n",
      "11900\n",
      "12000\n",
      "12000\n",
      "12000\n",
      "12100\n",
      "12200\n",
      "12300\n",
      "12400\n",
      "12400\n",
      "12400\n",
      "12400\n",
      "12500\n",
      "12500\n",
      "12600\n",
      "12700\n",
      "12800\n",
      "12800\n",
      "12900\n",
      "12900\n",
      "12900\n",
      "13000\n",
      "13100\n",
      "13200\n",
      "13200\n",
      "13300\n",
      "13300\n",
      "13300\n",
      "13400\n",
      "13500\n",
      "13600\n",
      "13700\n",
      "13800\n",
      "13800\n",
      "13800\n",
      "13900\n",
      "13900\n",
      "13900\n",
      "14000\n",
      "14000\n",
      "14000\n",
      "14100\n",
      "14100\n",
      "14200\n",
      "14200\n",
      "14300\n",
      "14300\n",
      "14300\n",
      "14400\n",
      "14500\n",
      "14600\n",
      "14700\n",
      "14700\n",
      "14800\n",
      "14800\n",
      "14900\n",
      "14900\n",
      "14900\n",
      "14900\n",
      "14900\n",
      "14900\n",
      "14900\n",
      "14900\n",
      "14900\n",
      "14900\n",
      "15000\n",
      "15000\n",
      "15000\n",
      "15100\n",
      "15100\n",
      "15200\n",
      "15200\n",
      "15300\n",
      "15300\n",
      "15400\n",
      "15400\n",
      "15500\n",
      "15500\n",
      "15600\n",
      "15600\n",
      "15600\n",
      "15600\n",
      "15700\n",
      "15800\n",
      "15900\n",
      "15900\n",
      "15900\n",
      "15900\n",
      "15900\n",
      "15900\n",
      "15900\n",
      "15900\n",
      "15900\n",
      "16000\n",
      "16000\n",
      "16000\n",
      "16000\n",
      "16000\n",
      "16000\n",
      "16100\n",
      "16100\n",
      "16100\n",
      "16100\n",
      "16100\n",
      "16100\n",
      "16100\n",
      "16100\n",
      "16100\n",
      "16100\n",
      "16200\n",
      "16300\n",
      "16400\n",
      "16500\n",
      "16500\n",
      "16500\n",
      "16600\n",
      "16600\n",
      "16600\n",
      "16700\n",
      "16700\n",
      "16700\n",
      "16800\n",
      "16800\n",
      "16800\n",
      "16800\n",
      "16900\n",
      "16900\n",
      "17000\n",
      "17100\n",
      "17100\n",
      "17100\n",
      "17200\n",
      "17300\n",
      "17300\n",
      "17300\n",
      "17300\n",
      "17400\n",
      "17400\n",
      "17500\n",
      "17500\n",
      "17500\n",
      "17500\n",
      "17500\n",
      "17500\n",
      "17500\n",
      "17600\n",
      "17700\n",
      "17700\n",
      "17700\n",
      "17700\n",
      "17700\n",
      "17700\n",
      "17800\n",
      "17900\n",
      "17900\n",
      "17900\n",
      "18000\n",
      "18000\n",
      "18100\n",
      "18200\n",
      "18200\n",
      "18200\n",
      "18300\n",
      "18300\n",
      "18400\n",
      "18400\n",
      "18500\n",
      "18500\n",
      "18500\n",
      "18600\n",
      "18600\n",
      "18700\n",
      "18700\n",
      "18700\n",
      "18700\n",
      "18800\n",
      "18800\n",
      "18800\n",
      "18800\n",
      "18900\n",
      "18900\n",
      "19000\n",
      "19100\n",
      "19100\n",
      "19100\n",
      "19200\n",
      "19200\n",
      "19200\n",
      "19300\n",
      "19400\n",
      "19500\n",
      "19500\n",
      "19500\n",
      "19500\n",
      "19500\n",
      "19600\n",
      "19700\n",
      "19700\n",
      "19800\n",
      "19800\n",
      "19900\n",
      "20000\n",
      "20100\n",
      "20200\n",
      "20300\n",
      "20400\n",
      "20400\n",
      "20400\n",
      "20400\n",
      "20500\n",
      "20600\n",
      "20600\n",
      "20700\n",
      "20700\n",
      "20800\n",
      "20800\n",
      "20900\n",
      "21000\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21100\n",
      "21200\n",
      "21300\n",
      "21400\n",
      "21400\n",
      "21500\n",
      "21500\n",
      "21500\n",
      "21500\n",
      "21500\n",
      "21500\n",
      "21500\n",
      "21600\n",
      "21700\n",
      "21700\n",
      "21700\n",
      "21800\n",
      "21800\n",
      "21900\n",
      "21900\n",
      "21900\n",
      "21900\n",
      "21900\n",
      "21900\n",
      "21900\n",
      "21900\n",
      "22000\n",
      "22000\n",
      "22100\n",
      "22100\n",
      "22100\n",
      "22100\n",
      "22200\n",
      "22200\n",
      "22300\n",
      "22300\n",
      "22300\n",
      "22300\n",
      "22400\n",
      "22500\n",
      "22600\n",
      "22600\n",
      "22700\n",
      "22700\n",
      "22800\n",
      "22800\n",
      "22800\n",
      "22800\n",
      "22800\n",
      "22800\n",
      "22900\n",
      "23000\n",
      "23100\n",
      "23200\n",
      "23300\n",
      "23400\n",
      "23400\n",
      "23400\n",
      "23400\n",
      "23400\n",
      "23400\n",
      "23400\n",
      "23400\n",
      "23500\n",
      "23600\n",
      "23600\n",
      "23600\n",
      "23600\n",
      "23700\n",
      "23800\n",
      "23900\n",
      "24000\n",
      "24100\n",
      "24200\n",
      "24300\n",
      "24300\n",
      "24300\n",
      "24400\n",
      "24400\n",
      "24400\n",
      "24500\n",
      "24600\n",
      "24600\n",
      "24700\n",
      "24700\n",
      "24700\n",
      "24800\n",
      "24800\n",
      "24900\n",
      "25000\n",
      "25000\n",
      "25000\n",
      "25100\n",
      "25100\n",
      "25100\n",
      "25100\n",
      "25200\n",
      "25200\n",
      "25300\n",
      "25400\n",
      "25400\n",
      "25500\n",
      "25500\n",
      "25500\n",
      "25600\n",
      "25800\n",
      "25800\n",
      "25800\n",
      "25900\n",
      "25900\n",
      "25900\n",
      "25900\n",
      "26000\n",
      "26100\n",
      "26100\n",
      "26100\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26200\n",
      "26300\n",
      "26300\n",
      "26300\n",
      "26300\n",
      "26300\n",
      "26400\n",
      "26500\n",
      "26600\n",
      "26700\n",
      "26800\n",
      "27000\n",
      "27100\n",
      "27100\n",
      "27100\n",
      "27200\n",
      "27200\n",
      "27200\n",
      "27300\n",
      "27300\n",
      "27400\n",
      "27400\n",
      "27400\n",
      "27400\n",
      "27500\n",
      "27600\n",
      "27600\n",
      "27700\n",
      "27800\n",
      "27800\n",
      "27900\n",
      "28000\n",
      "28000\n",
      "28100\n",
      "28200\n",
      "28200\n",
      "28300\n",
      "28400\n",
      "28400\n",
      "28500\n",
      "28600\n",
      "28600\n",
      "28700\n",
      "28700\n",
      "28700\n",
      "28700\n",
      "28700\n",
      "28800\n",
      "28900\n",
      "29000\n",
      "29100\n",
      "29200\n",
      "29300\n",
      "29400\n",
      "29500\n",
      "29600\n",
      "29600\n",
      "29800\n",
      "29800\n",
      "29900\n",
      "30000\n",
      "30100\n",
      "30100\n",
      "30100\n",
      "30100\n",
      "30200\n",
      "30200\n",
      "30300\n",
      "30400\n",
      "30400\n",
      "30400\n",
      "30500\n",
      "30500\n",
      "30500\n",
      "30600\n",
      "30700\n",
      "30700\n",
      "30700\n",
      "30700\n",
      "30700\n",
      "30700\n",
      "30700\n",
      "30800\n",
      "30800\n",
      "30800\n",
      "30800\n",
      "30800\n",
      "30800\n",
      "30900\n",
      "30900\n",
      "31000\n",
      "31000\n",
      "31000\n",
      "31000\n",
      "31100\n",
      "31100\n",
      "31200\n",
      "31300\n",
      "31300\n",
      "31300\n",
      "31400\n",
      "31500\n",
      "31500\n",
      "31500\n",
      "31500\n",
      "31600\n",
      "31700\n",
      "31800\n",
      "31900\n",
      "31900\n",
      "31900\n",
      "32000\n",
      "32000\n",
      "32000\n",
      "32100\n",
      "32100\n",
      "32200\n",
      "32300\n",
      "32400\n",
      "32400\n",
      "32400\n",
      "32400\n",
      "32500\n",
      "32500\n",
      "32600\n",
      "32600\n",
      "32700\n",
      "32700\n",
      "32800\n",
      "32800\n",
      "32800\n",
      "32800\n",
      "32800\n",
      "32800\n",
      "32800\n",
      "32900\n",
      "33000\n",
      "33000\n",
      "33000\n",
      "33000\n",
      "33000\n",
      "33100\n",
      "33200\n",
      "33200\n",
      "33200\n",
      "33300\n",
      "33400\n",
      "33500\n",
      "33500\n",
      "33600\n",
      "33600\n",
      "33600\n",
      "33700\n",
      "33700\n",
      "33800\n",
      "33900\n",
      "33900\n",
      "33900\n",
      "33900\n",
      "34000\n",
      "34100\n",
      "34200\n",
      "34200\n",
      "34200\n",
      "34300\n",
      "34300\n",
      "34400\n",
      "34400\n",
      "34400\n",
      "34500\n",
      "34500\n",
      "34600\n",
      "34600\n",
      "34700\n",
      "34700\n",
      "34800\n",
      "34800\n",
      "34900\n",
      "35000\n",
      "35000\n",
      "35000\n",
      "35000\n",
      "35000\n",
      "35000\n",
      "35000\n",
      "35000\n",
      "35000\n",
      "35000\n",
      "35100\n",
      "35100\n",
      "35100\n",
      "35200\n",
      "35200\n",
      "35200\n",
      "35300\n",
      "35400\n",
      "35400\n",
      "35500\n",
      "35500\n",
      "35500\n",
      "35500\n",
      "35500\n",
      "35600\n",
      "35700\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35800\n",
      "35900\n",
      "35900\n",
      "36000\n",
      "36000\n",
      "36100\n",
      "36100\n",
      "36100\n",
      "36100\n",
      "36100\n",
      "36200\n",
      "36200\n",
      "36300\n",
      "36400\n",
      "36500\n",
      "36600\n",
      "36600\n",
      "36700\n",
      "36800\n",
      "36800\n",
      "36800\n",
      "36800\n",
      "36800\n",
      "36900\n",
      "36900\n",
      "36900\n",
      "36900\n",
      "36900\n",
      "37000\n",
      "37000\n",
      "37000\n",
      "37100\n",
      "37100\n",
      "37100\n",
      "37100\n",
      "37200\n",
      "37200\n",
      "37200\n",
      "37300\n",
      "37300\n",
      "37400\n",
      "37500\n",
      "37500\n",
      "37500\n",
      "37600\n",
      "37600\n",
      "37600\n",
      "37600\n",
      "37700\n",
      "37700\n",
      "37800\n",
      "37900\n",
      "37900\n",
      "38000\n",
      "38000\n",
      "38100\n",
      "38200\n",
      "38200\n",
      "38300\n",
      "38300\n",
      "38300\n",
      "38400\n",
      "38400\n",
      "38400\n",
      "38500\n",
      "38600\n",
      "38600\n",
      "38700\n",
      "38800\n",
      "38800\n",
      "38800\n",
      "38900\n",
      "38900\n",
      "38900\n",
      "39000\n",
      "39100\n",
      "39200\n",
      "39200\n",
      "39300\n",
      "39300\n",
      "39400\n",
      "39400\n",
      "39400\n",
      "39500\n",
      "39600\n",
      "39600\n",
      "39600\n",
      "39600\n",
      "39600\n",
      "39600\n",
      "39700\n",
      "39700\n",
      "39700\n",
      "39700\n",
      "39800\n",
      "39800\n",
      "39800\n",
      "39800\n",
      "39800\n",
      "39800\n",
      "39800\n",
      "39900\n",
      "39900\n",
      "40000\n",
      "40000\n",
      "40000\n",
      "40000\n",
      "40000\n",
      "40000\n",
      "40000\n",
      "40000\n",
      "40000\n",
      "40000\n",
      "40000\n",
      "40100\n",
      "40100\n",
      "40100\n",
      "40200\n",
      "40200\n",
      "40200\n",
      "40200\n",
      "40300\n",
      "40300\n",
      "40300\n",
      "40300\n",
      "40400\n",
      "40500\n",
      "40600\n",
      "40700\n",
      "40700\n",
      "40800\n",
      "40900\n",
      "40900\n",
      "41000\n",
      "41100\n",
      "41100\n",
      "41100\n",
      "41200\n",
      "41300\n",
      "41300\n",
      "41400\n",
      "41400\n",
      "41400\n",
      "41400\n",
      "41500\n",
      "41600\n",
      "41700\n",
      "41700\n",
      "41800\n",
      "41800\n",
      "41800\n",
      "41900\n",
      "41900\n",
      "41900\n",
      "42000\n",
      "42100\n",
      "42100\n",
      "42100\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42200\n",
      "42300\n",
      "42300\n",
      "42300\n",
      "42400\n",
      "42500\n",
      "42500\n",
      "42500\n",
      "42500\n",
      "42500\n",
      "42500\n",
      "42600\n",
      "42600\n",
      "42700\n",
      "42800\n",
      "42900\n",
      "43000\n",
      "43100\n",
      "43100\n",
      "43200\n",
      "43200\n",
      "43200\n",
      "43200\n",
      "43300\n",
      "43400\n",
      "43500\n",
      "43500\n",
      "43600\n",
      "43600\n",
      "43600\n",
      "43600\n",
      "43600\n",
      "43600\n",
      "43700\n",
      "43700\n",
      "43800\n",
      "43800\n",
      "43800\n",
      "43900\n",
      "43900\n",
      "43900\n",
      "43900\n",
      "43900\n",
      "43900\n",
      "43900\n",
      "44000\n",
      "44000\n",
      "44100\n",
      "44100\n",
      "44200\n",
      "44300\n",
      "44300\n",
      "44300\n",
      "44300\n",
      "44300\n",
      "44300\n",
      "44400\n",
      "44500\n",
      "44500\n",
      "44500\n",
      "44600\n",
      "44600\n",
      "44600\n",
      "44600\n",
      "44700\n",
      "44700\n",
      "44700\n",
      "44700\n",
      "44800\n",
      "44800\n",
      "44900\n",
      "44900\n",
      "44900\n",
      "45000\n",
      "45000\n",
      "45000\n",
      "45000\n",
      "45100\n",
      "45100\n",
      "45100\n",
      "45200\n",
      "45200\n",
      "45300\n",
      "45300\n",
      "45300\n",
      "45400\n",
      "45400\n",
      "45400\n",
      "45500\n",
      "45500\n",
      "45600\n",
      "45600\n",
      "45600\n",
      "45600\n",
      "45600\n",
      "45700\n",
      "45800\n",
      "45900\n",
      "46000\n",
      "46000\n",
      "46000\n",
      "46200\n",
      "46300\n",
      "46400\n",
      "46500\n",
      "46600\n",
      "46700\n",
      "46700\n",
      "46700\n",
      "46800\n",
      "46900\n",
      "46900\n",
      "46900\n",
      "46900\n",
      "46900\n",
      "47000\n",
      "47100\n",
      "47100\n",
      "47100\n",
      "47200\n",
      "47200\n",
      "47300\n",
      "47300\n",
      "47300\n",
      "47300\n",
      "47300\n",
      "47300\n",
      "47300\n",
      "47400\n",
      "47400\n",
      "47500\n",
      "47600\n",
      "47700\n",
      "47700\n",
      "47700\n",
      "47800\n",
      "47800\n",
      "47900\n",
      "47900\n",
      "47900\n",
      "48000\n",
      "48000\n",
      "48100\n",
      "48100\n",
      "48200\n",
      "48200\n",
      "48300\n",
      "48300\n",
      "48400\n",
      "48500\n",
      "48600\n",
      "48700\n",
      "48700\n",
      "48700\n",
      "48700\n",
      "48800\n",
      "48800\n",
      "48900\n",
      "48900\n",
      "49000\n",
      "49100\n",
      "49100\n",
      "49100\n",
      "49300\n",
      "49300\n",
      "49400\n",
      "49400\n",
      "49500\n",
      "49500\n",
      "49600\n",
      "49600\n",
      "49600\n",
      "49700\n",
      "49800\n",
      "49800\n",
      "49800\n",
      "49800\n",
      "49800\n",
      "49900\n",
      "49900\n",
      "49900\n",
      "50000\n",
      "50000\n",
      "50100\n",
      "50200\n",
      "50300\n",
      "50300\n",
      "50400\n",
      "50400\n",
      "50400\n",
      "50400\n",
      "50400\n",
      "50400\n",
      "50400\n",
      "50400\n",
      "50500\n",
      "50500\n",
      "50600\n",
      "50700\n",
      "50700\n",
      "50800\n",
      "50800\n",
      "50800\n",
      "50900\n",
      "51000\n",
      "51000\n",
      "51100\n",
      "51100\n",
      "51200\n",
      "51300\n",
      "51300\n",
      "51400\n",
      "51500\n",
      "51600\n",
      "51600\n",
      "51600\n",
      "51600\n",
      "51700\n",
      "51800\n",
      "51900\n",
      "51900\n",
      "51900\n",
      "51900\n",
      "51900\n",
      "51900\n",
      "52000\n",
      "52000\n",
      "52000\n",
      "52100\n",
      "52100\n",
      "52100\n",
      "52100\n",
      "52100\n",
      "52100\n",
      "52100\n",
      "52200\n",
      "52200\n",
      "52200\n",
      "52300\n",
      "52300\n",
      "52400\n",
      "52400\n",
      "52500\n",
      "52600\n",
      "52600\n",
      "52600\n",
      "52600\n",
      "52700\n",
      "52800\n",
      "52800\n",
      "52900\n",
      "53000\n",
      "53000\n",
      "53100\n",
      "53100\n",
      "53100\n",
      "53100\n",
      "53100\n",
      "53100\n",
      "53200\n",
      "53300\n",
      "53400\n",
      "53400\n",
      "53500\n",
      "53500\n",
      "53500\n",
      "53500\n",
      "53500\n",
      "53600\n",
      "53700\n",
      "53700\n",
      "53800\n",
      "53900\n",
      "53900\n",
      "54000\n",
      "54000\n",
      "54000\n",
      "54000\n",
      "54000\n",
      "54100\n",
      "54200\n",
      "54200\n",
      "54300\n",
      "54300\n",
      "54400\n",
      "54500\n",
      "54500\n",
      "54500\n",
      "54600\n",
      "54600\n",
      "54600\n",
      "54600\n",
      "54700\n",
      "54800\n",
      "54900\n",
      "55000\n",
      "55000\n",
      "55100\n",
      "55100\n",
      "55200\n",
      "55200\n",
      "55300\n",
      "55400\n",
      "55500\n",
      "55500\n",
      "55600\n",
      "55700\n",
      "55700\n",
      "55700\n",
      "55800\n",
      "55800\n",
      "55900\n",
      "55900\n",
      "55900\n",
      "56000\n",
      "56000\n",
      "56100\n",
      "56100\n",
      "56100\n",
      "56200\n",
      "56200\n",
      "56300\n",
      "56300\n",
      "56300\n",
      "56300\n",
      "56300\n",
      "56400\n",
      "56400\n",
      "56500\n",
      "56500\n",
      "56500\n",
      "56600\n",
      "56600\n",
      "56700\n",
      "56800\n",
      "56900\n",
      "56900\n",
      "57000\n",
      "57000\n",
      "57000\n",
      "57000\n",
      "57000\n",
      "57000\n",
      "57000\n",
      "57100\n",
      "57100\n",
      "57100\n",
      "57100\n",
      "57100\n",
      "57200\n",
      "57300\n",
      "57400\n",
      "57500\n",
      "57600\n",
      "57700\n",
      "57800\n",
      "57800\n",
      "57800\n",
      "57800\n",
      "57800\n",
      "57900\n",
      "58000\n",
      "58100\n",
      "58200\n",
      "58200\n",
      "58200\n",
      "58200\n",
      "58300\n",
      "58300\n",
      "58300\n",
      "58300\n",
      "58300\n",
      "58400\n",
      "58400\n",
      "58400\n",
      "58500\n",
      "58500\n",
      "58600\n",
      "58700\n",
      "58800\n",
      "58800\n",
      "58900\n",
      "59000\n",
      "59000\n",
      "59100\n",
      "59100\n",
      "59100\n",
      "59100\n",
      "59100\n",
      "59200\n",
      "59300\n",
      "59400\n",
      "59500\n",
      "59500\n",
      "59600\n",
      "59700\n",
      "59700\n",
      "59800\n",
      "59800\n",
      "59900\n",
      "59900\n",
      "60000\n",
      "60100\n",
      "60100\n",
      "60200\n",
      "60300\n",
      "60300\n",
      "60300\n",
      "60400\n",
      "60500\n",
      "60600\n",
      "60600\n",
      "60700\n",
      "60700\n",
      "60700\n",
      "60800\n",
      "60900\n",
      "61000\n",
      "61100\n",
      "61100\n",
      "61200\n",
      "61200\n",
      "61300\n",
      "61300\n",
      "61400\n",
      "61400\n",
      "61500\n",
      "61500\n",
      "61600\n",
      "61600\n",
      "61600\n",
      "61700\n",
      "61700\n",
      "61800\n",
      "61800\n",
      "61800\n",
      "61800\n",
      "61800\n",
      "61900\n",
      "61900\n",
      "62000\n",
      "62100\n",
      "62100\n",
      "62200\n",
      "62200\n",
      "62300\n",
      "62300\n",
      "62300\n",
      "62400\n",
      "62500\n",
      "62500\n",
      "62600\n",
      "62600\n",
      "62600\n",
      "62600\n",
      "62700\n",
      "62700\n",
      "62700\n",
      "62700\n",
      "62800\n",
      "62900\n",
      "62900\n",
      "62900\n",
      "62900\n",
      "62900\n",
      "63000\n",
      "63100\n",
      "63200\n",
      "63200\n",
      "63200\n",
      "63300\n",
      "63300\n",
      "63400\n",
      "63500\n",
      "63500\n",
      "63500\n",
      "63500\n",
      "63600\n",
      "63700\n",
      "63700\n",
      "63700\n",
      "63800\n",
      "63800\n",
      "63900\n",
      "63900\n",
      "64000\n",
      "64100\n",
      "64200\n",
      "64200\n",
      "64300\n",
      "64300\n",
      "64300\n",
      "64300\n",
      "64300\n",
      "64400\n",
      "64500\n",
      "64600\n",
      "64700\n",
      "64800\n",
      "64800\n",
      "64800\n",
      "64900\n",
      "65000\n",
      "65000\n",
      "65000\n",
      "65100\n",
      "65200\n",
      "65200\n",
      "65300\n",
      "65300\n",
      "65400\n",
      "65500\n",
      "65600\n",
      "65600\n",
      "65600\n",
      "65700\n",
      "65700\n",
      "65700\n",
      "65800\n",
      "65800\n",
      "65900\n",
      "65900\n",
      "65900\n",
      "66000\n",
      "66100\n",
      "66100\n",
      "66200\n",
      "66300\n",
      "66400\n",
      "66500\n",
      "66600\n",
      "66600\n",
      "66700\n",
      "66700\n",
      "66800\n",
      "66900\n",
      "67000\n",
      "67100\n",
      "67200\n",
      "67300\n",
      "67300\n",
      "67400\n",
      "67400\n",
      "67500\n",
      "67500\n",
      "67600\n",
      "67600\n",
      "67700\n",
      "67800\n",
      "67900\n",
      "68000\n",
      "68000\n",
      "68100\n",
      "68100\n",
      "68200\n",
      "68200\n",
      "68300\n",
      "68400\n",
      "68500\n",
      "68600\n",
      "68700\n",
      "68800\n",
      "68900\n",
      "69000\n"
     ]
    }
   ],
   "source": [
    "final_pred=[]\n",
    "final_pred_score=[]\n",
    "for j in range(comments.shape[0]):\n",
    "    pred=[]\n",
    "    pred_score=[]\n",
    "    #j=269\n",
    "    xx=re.split(r'[.,;!\\n]', comments.loc[j][0])\n",
    "    \n",
    "    for k in range(len(xx)):\n",
    "        x=bigram[(preprocess(xx[k])).split()]\n",
    "        #print(x)\n",
    "        y=[0,0]\n",
    "        k=0\n",
    "        m=0\n",
    "        z2=0\n",
    "        yy=(\"fruit\" in x) + (\"veg\" in x) + (\"vegetable\" in x) \n",
    "        z=(\"banana\" in x)\n",
    "        bbq=(\"bbq\" in x) + (\"chicken\" in x)\n",
    "        meat = (\"meat\" in x)\n",
    "        organic = (\"organic\" in x)\n",
    "        chicken=(\"chicken\" in x)\n",
    "        deli = (\"deli\" in x)\n",
    "        rock = (\"rock\" in x)\n",
    "        seafood = (\"seafood\" in x)\n",
    "        \n",
    "        if (yy>0):\n",
    "            pred.append('PRODUCE')\n",
    "        elif (z==1):    \n",
    "            pred.append('PRODUCE')\n",
    "        elif(bbq==2):\n",
    "            pred.append('SERVICED MEAT')\n",
    "        elif (meat==1):\n",
    "            pred.append('SERVICED MEAT')\n",
    "        elif(organic==1):\n",
    "            pred.append('ORGANICS')\n",
    "        elif(chicken==1):\n",
    "            pred.append('FRESH POULTRY')\n",
    "        elif((deli==1) and (rock==0)):\n",
    "            pred.append('DELI')\n",
    "        elif(seafood==1):\n",
    "            pred.append('SEAFOOD')\n",
    "        else:\n",
    "            #pred_subcat2.append(comments.loc[j]['predicted_subcat'])\n",
    "            if(j%100==0):\n",
    "                print(j)\n",
    "                #print(len(pred))\n",
    "            if(len(x)>0):\n",
    "\n",
    "                for i in range(len(x)):\n",
    "                    try:\n",
    "                        z2=z2+w2v_model[x[i]].reshape(-1,100)\n",
    "                        #y=y+model2.predict_proba(model[x[i]].reshape(-1,100))\n",
    "                        m=m+1\n",
    "                        #print(m)\n",
    "                    except KeyError:\n",
    "                        k=k+1\n",
    "\n",
    "                if(m>0):\n",
    "                    z=model.predict(z2/m)\n",
    "                else:\n",
    "                    z=[0]\n",
    "                #print(comments.loc[j][0])\n",
    "            else:\n",
    "                z=[0]\n",
    "        #print(z)\n",
    "        try:\n",
    "\n",
    "\n",
    "            pred.append(subcat_index_mapping[subcat_index_mapping['make_code']==np.argmax(z,axis=1)[0]]['subcat'].item())\n",
    "            #print(subcat_index_mapping[subcat_index_mapping['make_code']==np.argmax(z,axis=1)[0]]['subcat'].item())\n",
    "            #print(subcat_index_mapping[subcat_index_mapping['make_code']==np.argmax(z,axis=1)[0]]['subcat'].item())\n",
    "            if(subcat_index_mapping[subcat_index_mapping['make_code']==np.argmax(z,axis=1)[0]]['subcat'].item()==\"unknown\"):\n",
    "                pred_score.append(0.01)\n",
    "            else:\n",
    "\n",
    "                pred_score.append(z[0][np.argmax(z,axis=1)][0])\n",
    "\n",
    "        except:\n",
    "            pred.append(\"unknown2\")\n",
    "            pred_score.append(0)\n",
    "    #print(pred)\n",
    "    xxx=', '.join(set([word for word in pred]))\n",
    "    yyy=', '.join(set([str(word) for word in pred_score if word>0.0]))\n",
    "    final_pred.append(xxx)\n",
    "    final_pred_score.append(yyy)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "comments=pd.read_table(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/comments.csv\",encoding='latin1')\n",
    "comments['predicted_subcat']=final_pred\n",
    "comments['predicted_subcat_score']=final_pred_score\n",
    "comments.to_csv(\"//QATLPCFS001/Users/akishore/Desktop/DL/voc/comments_predicted_subcats36.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Comment</th>\n",
       "      <th>predicted_subcat</th>\n",
       "      <th>predicted_subcat_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mainland cheddar not available</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "      <td>0.525254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>no cholesterol reducing cheese slices</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "      <td>0.382147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>didnt have enough Kraft cheese spread light</td>\n",
       "      <td>DAIRY - BUTTER &amp; MARGARINE</td>\n",
       "      <td>0.584017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>No Feta cheese.</td>\n",
       "      <td>unknown2, CHEESE PRE-PACKED COOKING</td>\n",
       "      <td>0.367359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Toffutti cream cheese has been deleted</td>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "      <td>0.409461</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Comment  \\\n",
       "0               Mainland cheddar not available   \n",
       "1        no cholesterol reducing cheese slices   \n",
       "2  didnt have enough Kraft cheese spread light   \n",
       "3                              No Feta cheese.   \n",
       "4       Toffutti cream cheese has been deleted   \n",
       "\n",
       "                      predicted_subcat predicted_subcat_score  \n",
       "0                       DAIRY - CHEESE               0.525254  \n",
       "1                       DAIRY - CHEESE               0.382147  \n",
       "2           DAIRY - BUTTER & MARGARINE               0.584017  \n",
       "3  unknown2, CHEESE PRE-PACKED COOKING               0.367359  \n",
       "4                       DAIRY - CHEESE               0.409461  "
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 991,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-08 10:24:14,888 : INFO : saving Word2Vec object under word2vec_final_voc, separately None\n",
      "2017-11-08 10:24:14,898 : INFO : not storing attribute syn0norm\n",
      "2017-11-08 10:24:14,899 : INFO : not storing attribute cum_table\n",
      "2017-11-08 10:24:14,989 : INFO : saved word2vec_final_voc\n"
     ]
    }
   ],
   "source": [
    "w2v_model.save(\"word2vec_final_voc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 992,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('my_model_voc.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1039,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>dvd</td>\n",
       "      <td>0.024516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>card</td>\n",
       "      <td>0.015206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>chicken</td>\n",
       "      <td>0.014172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>black</td>\n",
       "      <td>0.013914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>cheese</td>\n",
       "      <td>0.012828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>beef</td>\n",
       "      <td>0.011941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>blue</td>\n",
       "      <td>0.010523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>bk</td>\n",
       "      <td>0.010258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>red</td>\n",
       "      <td>0.010113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>white</td>\n",
       "      <td>0.010015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>choc</td>\n",
       "      <td>0.009767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>bag</td>\n",
       "      <td>0.009100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>organic</td>\n",
       "      <td>0.008321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>cream</td>\n",
       "      <td>0.008299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>milk</td>\n",
       "      <td>0.008121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>prepack</td>\n",
       "      <td>0.007958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>green</td>\n",
       "      <td>0.007435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>apple</td>\n",
       "      <td>0.007191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>baby</td>\n",
       "      <td>0.007171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>wow</td>\n",
       "      <td>0.007145</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        term    weight\n",
       "64       dvd  0.024516\n",
       "34      card  0.015206\n",
       "39   chicken  0.014172\n",
       "17     black  0.013914\n",
       "38    cheese  0.012828\n",
       "12      beef  0.011941\n",
       "20      blue  0.010523\n",
       "16        bk  0.010258\n",
       "156      red  0.010113\n",
       "201    white  0.010015\n",
       "43      choc  0.009767\n",
       "6        bag  0.009100\n",
       "129  organic  0.008321\n",
       "52     cream  0.008299\n",
       "111     milk  0.008121\n",
       "149  prepack  0.007958\n",
       "81     green  0.007435\n",
       "1      apple  0.007191\n",
       "4       baby  0.007171\n",
       "203      wow  0.007145"
      ]
     },
     "execution_count": 1039,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tvec = TfidfVectorizer(min_df=.0025, max_df=.1, stop_words='english')\n",
    "tvec_weights = tvec.fit_transform(data['ARTICLE_NAME'])\n",
    "weights = np.asarray(tvec_weights.mean(axis=0)).ravel().tolist()\n",
    "weights_df = pd.DataFrame({'term': tvec.get_feature_names(), 'weight': weights})\n",
    "weights_df.sort_values(by='weight', ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1040,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "weights_df=weights_df.reset_index()\n",
    "weights_df=weights_df.drop(['index'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARTICLE_NAME</th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7130</th>\n",
       "      <td>heinz lk shredz fruit veg</td>\n",
       "      <td>BABY FOOD</td>\n",
       "      <td>gc baby apple custad age golden circle baby fd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8174</th>\n",
       "      <td>vegetali fruit caramelsed pear</td>\n",
       "      <td>DAIRY - SNACKS KIDS</td>\n",
       "      <td>weight watcher lemon deliciou yoplait blueberr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8522</th>\n",
       "      <td>nippy fruit nippy frveg</td>\n",
       "      <td>DAIRY - CHILLED JUICES &amp; DRINKS</td>\n",
       "      <td>lt apple juice supreme orange nectar juice chi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13900</th>\n",
       "      <td>juice fruit veg citru splash</td>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16120</th>\n",
       "      <td>vegetali fruit rhubarb blubery</td>\n",
       "      <td>DAIRY - SNACKS KIDS</td>\n",
       "      <td>weight watcher lemon deliciou yoplait blueberr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17056</th>\n",
       "      <td>gc fruit veg jc pinepunch mnth</td>\n",
       "      <td>BABY FOOD</td>\n",
       "      <td>gc baby apple custad age golden circle baby fd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21426</th>\n",
       "      <td>fruit veg juice apple plum fusn</td>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21680</th>\n",
       "      <td>fruit veg breakfast</td>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29709</th>\n",
       "      <td>artisse fruit veg bar app rspbry</td>\n",
       "      <td>HEALTH FOODS</td>\n",
       "      <td>sunrice brown rice chip parm tom macro apr cur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34540</th>\n",
       "      <td>artisse fruit veg bar app raisin</td>\n",
       "      <td>HEALTH FOODS</td>\n",
       "      <td>sunrice brown rice chip parm tom macro apr cur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39443</th>\n",
       "      <td>prime raw chickn fruit veg</td>\n",
       "      <td>PETFOOD</td>\n",
       "      <td>maranui steak rabbit pottle butch white jimbo ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40989</th>\n",
       "      <td>fruit veg juice energy</td>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42957</th>\n",
       "      <td>fruit veg juice tropical</td>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44799</th>\n",
       "      <td>juice fruit vegtropical</td>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50436</th>\n",
       "      <td>raffertysgrd fruit veg pear bar</td>\n",
       "      <td>BABY FOOD</td>\n",
       "      <td>gc baby apple custad age golden circle baby fd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62593</th>\n",
       "      <td>creative gourmet fruit veg cube</td>\n",
       "      <td>FREEZER - DESSERTS &amp; PASTRY</td>\n",
       "      <td>nanna danish apple crofter black forest bavari...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63732</th>\n",
       "      <td>vegetali fruit rhubarb apple</td>\n",
       "      <td>DAIRY - SNACKS KIDS</td>\n",
       "      <td>weight watcher lemon deliciou yoplait blueberr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70495</th>\n",
       "      <td>hnz kid fruit veg peach appl veg</td>\n",
       "      <td>BABY FOOD</td>\n",
       "      <td>gc baby apple custad age golden circle baby fd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70783</th>\n",
       "      <td>juice fruit veg tropical</td>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71487</th>\n",
       "      <td>fruit veg jc cit crush mnth</td>\n",
       "      <td>BABY FOOD</td>\n",
       "      <td>gc baby apple custad age golden circle baby fd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73494</th>\n",
       "      <td>vegetali fruit poachfig syrup</td>\n",
       "      <td>DAIRY - SNACKS KIDS</td>\n",
       "      <td>weight watcher lemon deliciou yoplait blueberr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79101</th>\n",
       "      <td>fruit veg apple berry fusion</td>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82535</th>\n",
       "      <td>fruit veg cranberry apple</td>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87700</th>\n",
       "      <td>anny veggi fruit leather</td>\n",
       "      <td>unknown</td>\n",
       "      <td>co mega val groovy disney princess role play a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121831</th>\n",
       "      <td>juice fruit veg citru splash</td>\n",
       "      <td>unknown</td>\n",
       "      <td>co mega val groovy disney princess role play a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128279</th>\n",
       "      <td>juice fruit veg tropical combi</td>\n",
       "      <td>unknown</td>\n",
       "      <td>co mega val groovy disney princess role play a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139515</th>\n",
       "      <td>smart juice tropicalfruit veg</td>\n",
       "      <td>unknown</td>\n",
       "      <td>co mega val groovy disney princess role play a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143275</th>\n",
       "      <td>fruit veg juice melon burst</td>\n",
       "      <td>unknown</td>\n",
       "      <td>co mega val groovy disney princess role play a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            ARTICLE_NAME                      SUBCAT_NAME  \\\n",
       "7130           heinz lk shredz fruit veg                        BABY FOOD   \n",
       "8174      vegetali fruit caramelsed pear              DAIRY - SNACKS KIDS   \n",
       "8522             nippy fruit nippy frveg  DAIRY - CHILLED JUICES & DRINKS   \n",
       "13900       juice fruit veg citru splash          FRUIT JUICE - LONG LIFE   \n",
       "16120     vegetali fruit rhubarb blubery              DAIRY - SNACKS KIDS   \n",
       "17056     gc fruit veg jc pinepunch mnth                        BABY FOOD   \n",
       "21426    fruit veg juice apple plum fusn          FRUIT JUICE - LONG LIFE   \n",
       "21680                fruit veg breakfast          FRUIT JUICE - LONG LIFE   \n",
       "29709   artisse fruit veg bar app rspbry                     HEALTH FOODS   \n",
       "34540   artisse fruit veg bar app raisin                     HEALTH FOODS   \n",
       "39443         prime raw chickn fruit veg                          PETFOOD   \n",
       "40989             fruit veg juice energy          FRUIT JUICE - LONG LIFE   \n",
       "42957           fruit veg juice tropical          FRUIT JUICE - LONG LIFE   \n",
       "44799            juice fruit vegtropical          FRUIT JUICE - LONG LIFE   \n",
       "50436    raffertysgrd fruit veg pear bar                        BABY FOOD   \n",
       "62593    creative gourmet fruit veg cube      FREEZER - DESSERTS & PASTRY   \n",
       "63732       vegetali fruit rhubarb apple              DAIRY - SNACKS KIDS   \n",
       "70495   hnz kid fruit veg peach appl veg                        BABY FOOD   \n",
       "70783           juice fruit veg tropical          FRUIT JUICE - LONG LIFE   \n",
       "71487        fruit veg jc cit crush mnth                        BABY FOOD   \n",
       "73494      vegetali fruit poachfig syrup              DAIRY - SNACKS KIDS   \n",
       "79101       fruit veg apple berry fusion          FRUIT JUICE - LONG LIFE   \n",
       "82535          fruit veg cranberry apple          FRUIT JUICE - LONG LIFE   \n",
       "87700           anny veggi fruit leather                          unknown   \n",
       "121831      juice fruit veg citru splash                          unknown   \n",
       "128279    juice fruit veg tropical combi                          unknown   \n",
       "139515     smart juice tropicalfruit veg                          unknown   \n",
       "143275       fruit veg juice melon burst                          unknown   \n",
       "\n",
       "                                                     text  \n",
       "7130    gc baby apple custad age golden circle baby fd...  \n",
       "8174    weight watcher lemon deliciou yoplait blueberr...  \n",
       "8522    lt apple juice supreme orange nectar juice chi...  \n",
       "13900   dole juice mango mambo juce strawberry kiwifru...  \n",
       "16120   weight watcher lemon deliciou yoplait blueberr...  \n",
       "17056   gc baby apple custad age golden circle baby fd...  \n",
       "21426   dole juice mango mambo juce strawberry kiwifru...  \n",
       "21680   dole juice mango mambo juce strawberry kiwifru...  \n",
       "29709   sunrice brown rice chip parm tom macro apr cur...  \n",
       "34540   sunrice brown rice chip parm tom macro apr cur...  \n",
       "39443   maranui steak rabbit pottle butch white jimbo ...  \n",
       "40989   dole juice mango mambo juce strawberry kiwifru...  \n",
       "42957   dole juice mango mambo juce strawberry kiwifru...  \n",
       "44799   dole juice mango mambo juce strawberry kiwifru...  \n",
       "50436   gc baby apple custad age golden circle baby fd...  \n",
       "62593   nanna danish apple crofter black forest bavari...  \n",
       "63732   weight watcher lemon deliciou yoplait blueberr...  \n",
       "70495   gc baby apple custad age golden circle baby fd...  \n",
       "70783   dole juice mango mambo juce strawberry kiwifru...  \n",
       "71487   gc baby apple custad age golden circle baby fd...  \n",
       "73494   weight watcher lemon deliciou yoplait blueberr...  \n",
       "79101   dole juice mango mambo juce strawberry kiwifru...  \n",
       "82535   dole juice mango mambo juce strawberry kiwifru...  \n",
       "87700   co mega val groovy disney princess role play a...  \n",
       "121831  co mega val groovy disney princess role play a...  \n",
       "128279  co mega val groovy disney princess role play a...  \n",
       "139515  co mega val groovy disney princess role play a...  \n",
       "143275  co mega val groovy disney princess role play a...  "
      ]
     },
     "execution_count": 1135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x=\"organic\"\n",
    "data[data['ARTICLE_NAME'].str.contains('fruit') & data['ARTICLE_NAME'].str.contains('veg')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1058,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n"
     ]
    }
   ],
   "source": [
    "article_count=[]\n",
    "article_subcat_name=[]\n",
    "subcat_count=[]\n",
    "subcat_name=[]\n",
    "for i in range(weights_df.shape[0]):\n",
    "    print(i)\n",
    "    x=weights_df.loc[i]['term']\n",
    "    #print(x)\n",
    "    art_len=len(data[data['ARTICLE_NAME'].str.contains(x.lower())]['SUBCAT_NAME'].value_counts())\n",
    "    subcat_len=len(data[data['SUBCAT_NAME'].str.contains(x.upper())]['SUBCAT_NAME'].value_counts())\n",
    "    if(art_len>0):\n",
    "        \n",
    "        article_count.append(data[data['ARTICLE_NAME'].str.contains(x.lower())]['SUBCAT_NAME'].value_counts()[0])\n",
    "        article_subcat_name.append(data[data['ARTICLE_NAME'].str.contains(x.lower())]['SUBCAT_NAME'].value_counts().index[0])\n",
    "        \n",
    "    else:\n",
    "        article_count.append(0)\n",
    "        article_subcat_name.append('unknown')\n",
    "    \n",
    "    if(subcat_len>0):\n",
    "        \n",
    "        subcat_count.append(data[data['SUBCAT_NAME'].str.contains(x.upper())]['SUBCAT_NAME'].value_counts()[0])\n",
    "        subcat_name.append(data[data['SUBCAT_NAME'].str.contains(x.upper())]['SUBCAT_NAME'].value_counts().index[0])\n",
    "        \n",
    "    else:\n",
    "        subcat_count.append(0)\n",
    "        subcat_name.append('unknown')\n",
    "    #subcat_count.append(data[data['SUBCAT_NAME'].str.contains(x.upper())]['SUBCAT_NAME'].value_counts()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1054,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "210"
      ]
     },
     "execution_count": 1054,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(article_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1057,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['DAIRY - YOGHURT', 'unknown', 'FREEZER - ICE CREAM TAKE HOME',\n",
       "       'DAIRY - SNACKS KIDS', 'MUESLI BARS', 'HEALTH FOODS', 'DAIRY - DIPS',\n",
       "       'BISCUITS - PLAIN & FANCY', 'DELI - CHILLED', 'SNACK - NUTS',\n",
       "       'DAIRY - QUICK & EASY MEALS', 'DAIRY - CHILLED JUICES & DRINKS',\n",
       "       'FREEZER - ICE CREAM SINGLES', 'BABY FOOD',\n",
       "       'FREEZER - DESSERTS & PASTRY', 'BREAKFAST - MUESLI & OATS',\n",
       "       'FREEZER - ICE CREAM MULTIPACKS', 'CONFECTIONERY - SUGAR',\n",
       "       'CONFECTIONERY - NOVELTY', 'DAIRY - SNACKS ADULT', 'DAIRY - CHEESE',\n",
       "       'CHEESE PRE-PACKED ENTERTAINING', 'PETFOOD', 'SALADS PREPACKED',\n",
       "       'RESALE CAKE'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 1057,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['ARTICLE_NAME'].str.contains(x.lower())]['SUBCAT_NAME'].value_counts().index[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unknown                            817\n",
       "FRESH POULTRY                      808\n",
       "HOT FOOD                           332\n",
       "FREEZER - POULTRY                  274\n",
       "FREEZER - MEALS                    186\n",
       "PET NEEDS - CAT FOOD WET           146\n",
       "DAIRY - QUICK & EASY MEALS         144\n",
       "DAIRY - SMALLGOODS                 118\n",
       "SUSHI                              108\n",
       "NOODLES                             99\n",
       "PREPARED MEALS                      67\n",
       "CHIPS - SHARING                     67\n",
       "PET NEEDS - DOG FOOD WET            58\n",
       "PET NEEDS - DOG FOOD DRY            56\n",
       "PET NEEDS - DOG TREATS              50\n",
       "SOUPS                               48\n",
       "PETFOOD                             45\n",
       "ASIAN FOODS                         35\n",
       "BISCUITS - SNACKING                 33\n",
       "BABY FOOD                           33\n",
       "PET NEEDS - CAT FOOD DRY            32\n",
       "SALADS PREPACKED                    27\n",
       "CANNED FISH                         20\n",
       "CHEESE PRE-PACKED ENTERTAINING      16\n",
       "CHIPS - MULTIPACKS                  15\n",
       "FREEZER - PIZZA                     15\n",
       "HEALTH FOODS                        13\n",
       "SPREADS - OTHER                      9\n",
       "DAIRY - DIPS                         8\n",
       "RICE                                 7\n",
       "FREEZER - AUTHENTIC ASIAN            7\n",
       "FREEZER - PET FOOD                   7\n",
       "FREEZER - GLUTEN FREE                6\n",
       "PIZZAS PREPACKED                     6\n",
       "ORGANIC MEAT                         6\n",
       "BEEF CASE READY                      5\n",
       "DAIRY - CHEESE                       5\n",
       "FREEZER - VEGETABLES                 4\n",
       "SAUCES                               4\n",
       "DELI - CHILLED                       3\n",
       "BISCUITS - CRISPBREAD & CRACKER      3\n",
       "CONFECTIONERY - SUGAR                2\n",
       "SNACK - NUTS                         2\n",
       "PASTA SAUCE & CHEESE                 2\n",
       "MUESLI BARS                          1\n",
       "DAIRY - SNACKS KIDS                  1\n",
       "Name: SUBCAT_NAME, dtype: int64"
      ]
     },
     "execution_count": 1125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['ARTICLE_NAME'].str.contains(\"chicken\")]['SUBCAT_NAME'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "110"
      ]
     },
     "execution_count": 1129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(model.predict(w2v_model['chicken'].reshape(-1,100)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1022,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARTICLE_NAME</th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>macro organic beef blade steak</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>org nectarine white</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>org garlic</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736</th>\n",
       "      <td>org pumpkin jarrahdal</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>751</th>\n",
       "      <td>belmore organic rib eye steak rw</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>805</th>\n",
       "      <td>macro organic lamb forequarterchop</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883</th>\n",
       "      <td>org carrot petite</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1035</th>\n",
       "      <td>org leek</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1231</th>\n",
       "      <td>org lebanese cucumber</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1234</th>\n",
       "      <td>organic beef sandwchsteak</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1358</th>\n",
       "      <td>org orange valencium</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1390</th>\n",
       "      <td>cleaver beef mini meatball</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1438</th>\n",
       "      <td>organic loose cabbage</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1488</th>\n",
       "      <td>organic beef stir fry</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1495</th>\n",
       "      <td>steak bbq organic</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1528</th>\n",
       "      <td>org spring onion</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1623</th>\n",
       "      <td>macro organic lamb diced min</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1634</th>\n",
       "      <td>org grape</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1712</th>\n",
       "      <td>org lychee</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2254</th>\n",
       "      <td>org orange valencium</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2309</th>\n",
       "      <td>org pear nashi green</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>org beetroot</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2488</th>\n",
       "      <td>org apple royal gala</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2492</th>\n",
       "      <td>org garlic loose</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2859</th>\n",
       "      <td>org pear pp</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2898</th>\n",
       "      <td>org conv tomato</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3211</th>\n",
       "      <td>org corn cobbette</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3238</th>\n",
       "      <td>macro organic beef sau tom basil</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3293</th>\n",
       "      <td>organic nut macadamium rstd</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3453</th>\n",
       "      <td>org nectarin yellow</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80523</th>\n",
       "      <td>org swede</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80526</th>\n",
       "      <td>org grape loose</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80606</th>\n",
       "      <td>org plum</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80730</th>\n",
       "      <td>inglewood chkn orgncwing</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80936</th>\n",
       "      <td>cleaver org lamb honey mint riblet</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81120</th>\n",
       "      <td>org lime loose</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81418</th>\n",
       "      <td>org grapefruit</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81480</th>\n",
       "      <td>org cucumber green</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81496</th>\n",
       "      <td>organic herb rosemary bunch</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81600</th>\n",
       "      <td>org kiwifruit</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81611</th>\n",
       "      <td>org cabbage green</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81638</th>\n",
       "      <td>org loose courgette</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81725</th>\n",
       "      <td>org conv apple fuji</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81866</th>\n",
       "      <td>org carrot</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81873</th>\n",
       "      <td>org broccolini</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82014</th>\n",
       "      <td>org silverbeet</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82259</th>\n",
       "      <td>org grape lunch</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82314</th>\n",
       "      <td>org pear beurre bosc</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82346</th>\n",
       "      <td>org beetroot</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82411</th>\n",
       "      <td>lamb leg organic</td>\n",
       "      <td>ORGANIC MEAT</td>\n",
       "      <td>macro organic beef blade steak belmore organic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82423</th>\n",
       "      <td>org capsicum loose</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82533</th>\n",
       "      <td>org coconut drinking</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82600</th>\n",
       "      <td>org ginger</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82675</th>\n",
       "      <td>org apple red del</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82739</th>\n",
       "      <td>org plum</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82841</th>\n",
       "      <td>org onion spnish red</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83107</th>\n",
       "      <td>org brussel sprout gpp</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83220</th>\n",
       "      <td>organic avocado</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83309</th>\n",
       "      <td>org conv lettuce baby co</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83498</th>\n",
       "      <td>org pumpkin butternut</td>\n",
       "      <td>ORGANICS</td>\n",
       "      <td>org nectarine white org garlic org pumpkin jar...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>876 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             ARTICLE_NAME   SUBCAT_NAME  \\\n",
       "43         macro organic beef blade steak  ORGANIC MEAT   \n",
       "436                   org nectarine white      ORGANICS   \n",
       "488                            org garlic      ORGANICS   \n",
       "736                 org pumpkin jarrahdal      ORGANICS   \n",
       "751      belmore organic rib eye steak rw  ORGANIC MEAT   \n",
       "805    macro organic lamb forequarterchop  ORGANIC MEAT   \n",
       "883                     org carrot petite      ORGANICS   \n",
       "1035                             org leek      ORGANICS   \n",
       "1231                org lebanese cucumber      ORGANICS   \n",
       "1234            organic beef sandwchsteak  ORGANIC MEAT   \n",
       "1358                 org orange valencium      ORGANICS   \n",
       "1390           cleaver beef mini meatball  ORGANIC MEAT   \n",
       "1438                organic loose cabbage      ORGANICS   \n",
       "1488                organic beef stir fry  ORGANIC MEAT   \n",
       "1495                    steak bbq organic  ORGANIC MEAT   \n",
       "1528                     org spring onion      ORGANICS   \n",
       "1623         macro organic lamb diced min  ORGANIC MEAT   \n",
       "1634                            org grape      ORGANICS   \n",
       "1712                           org lychee      ORGANICS   \n",
       "2254                 org orange valencium      ORGANICS   \n",
       "2309                 org pear nashi green      ORGANICS   \n",
       "2485                         org beetroot      ORGANICS   \n",
       "2488                 org apple royal gala      ORGANICS   \n",
       "2492                     org garlic loose      ORGANICS   \n",
       "2859                          org pear pp      ORGANICS   \n",
       "2898                      org conv tomato      ORGANICS   \n",
       "3211                    org corn cobbette      ORGANICS   \n",
       "3238     macro organic beef sau tom basil  ORGANIC MEAT   \n",
       "3293          organic nut macadamium rstd      ORGANICS   \n",
       "3453                  org nectarin yellow      ORGANICS   \n",
       "...                                   ...           ...   \n",
       "80523                           org swede      ORGANICS   \n",
       "80526                     org grape loose      ORGANICS   \n",
       "80606                            org plum      ORGANICS   \n",
       "80730            inglewood chkn orgncwing  ORGANIC MEAT   \n",
       "80936  cleaver org lamb honey mint riblet  ORGANIC MEAT   \n",
       "81120                      org lime loose      ORGANICS   \n",
       "81418                      org grapefruit      ORGANICS   \n",
       "81480                  org cucumber green      ORGANICS   \n",
       "81496         organic herb rosemary bunch      ORGANICS   \n",
       "81600                       org kiwifruit      ORGANICS   \n",
       "81611                   org cabbage green      ORGANICS   \n",
       "81638                 org loose courgette      ORGANICS   \n",
       "81725                 org conv apple fuji      ORGANICS   \n",
       "81866                          org carrot      ORGANICS   \n",
       "81873                      org broccolini      ORGANICS   \n",
       "82014                      org silverbeet      ORGANICS   \n",
       "82259                     org grape lunch      ORGANICS   \n",
       "82314                org pear beurre bosc      ORGANICS   \n",
       "82346                        org beetroot      ORGANICS   \n",
       "82411                    lamb leg organic  ORGANIC MEAT   \n",
       "82423                  org capsicum loose      ORGANICS   \n",
       "82533                org coconut drinking      ORGANICS   \n",
       "82600                          org ginger      ORGANICS   \n",
       "82675                   org apple red del      ORGANICS   \n",
       "82739                            org plum      ORGANICS   \n",
       "82841                org onion spnish red      ORGANICS   \n",
       "83107              org brussel sprout gpp      ORGANICS   \n",
       "83220                     organic avocado      ORGANICS   \n",
       "83309            org conv lettuce baby co      ORGANICS   \n",
       "83498               org pumpkin butternut      ORGANICS   \n",
       "\n",
       "                                                    text  \n",
       "43     macro organic beef blade steak belmore organic...  \n",
       "436    org nectarine white org garlic org pumpkin jar...  \n",
       "488    org nectarine white org garlic org pumpkin jar...  \n",
       "736    org nectarine white org garlic org pumpkin jar...  \n",
       "751    macro organic beef blade steak belmore organic...  \n",
       "805    macro organic beef blade steak belmore organic...  \n",
       "883    org nectarine white org garlic org pumpkin jar...  \n",
       "1035   org nectarine white org garlic org pumpkin jar...  \n",
       "1231   org nectarine white org garlic org pumpkin jar...  \n",
       "1234   macro organic beef blade steak belmore organic...  \n",
       "1358   org nectarine white org garlic org pumpkin jar...  \n",
       "1390   macro organic beef blade steak belmore organic...  \n",
       "1438   org nectarine white org garlic org pumpkin jar...  \n",
       "1488   macro organic beef blade steak belmore organic...  \n",
       "1495   macro organic beef blade steak belmore organic...  \n",
       "1528   org nectarine white org garlic org pumpkin jar...  \n",
       "1623   macro organic beef blade steak belmore organic...  \n",
       "1634   org nectarine white org garlic org pumpkin jar...  \n",
       "1712   org nectarine white org garlic org pumpkin jar...  \n",
       "2254   org nectarine white org garlic org pumpkin jar...  \n",
       "2309   org nectarine white org garlic org pumpkin jar...  \n",
       "2485   org nectarine white org garlic org pumpkin jar...  \n",
       "2488   org nectarine white org garlic org pumpkin jar...  \n",
       "2492   org nectarine white org garlic org pumpkin jar...  \n",
       "2859   org nectarine white org garlic org pumpkin jar...  \n",
       "2898   org nectarine white org garlic org pumpkin jar...  \n",
       "3211   org nectarine white org garlic org pumpkin jar...  \n",
       "3238   macro organic beef blade steak belmore organic...  \n",
       "3293   org nectarine white org garlic org pumpkin jar...  \n",
       "3453   org nectarine white org garlic org pumpkin jar...  \n",
       "...                                                  ...  \n",
       "80523  org nectarine white org garlic org pumpkin jar...  \n",
       "80526  org nectarine white org garlic org pumpkin jar...  \n",
       "80606  org nectarine white org garlic org pumpkin jar...  \n",
       "80730  macro organic beef blade steak belmore organic...  \n",
       "80936  macro organic beef blade steak belmore organic...  \n",
       "81120  org nectarine white org garlic org pumpkin jar...  \n",
       "81418  org nectarine white org garlic org pumpkin jar...  \n",
       "81480  org nectarine white org garlic org pumpkin jar...  \n",
       "81496  org nectarine white org garlic org pumpkin jar...  \n",
       "81600  org nectarine white org garlic org pumpkin jar...  \n",
       "81611  org nectarine white org garlic org pumpkin jar...  \n",
       "81638  org nectarine white org garlic org pumpkin jar...  \n",
       "81725  org nectarine white org garlic org pumpkin jar...  \n",
       "81866  org nectarine white org garlic org pumpkin jar...  \n",
       "81873  org nectarine white org garlic org pumpkin jar...  \n",
       "82014  org nectarine white org garlic org pumpkin jar...  \n",
       "82259  org nectarine white org garlic org pumpkin jar...  \n",
       "82314  org nectarine white org garlic org pumpkin jar...  \n",
       "82346  org nectarine white org garlic org pumpkin jar...  \n",
       "82411  macro organic beef blade steak belmore organic...  \n",
       "82423  org nectarine white org garlic org pumpkin jar...  \n",
       "82533  org nectarine white org garlic org pumpkin jar...  \n",
       "82600  org nectarine white org garlic org pumpkin jar...  \n",
       "82675  org nectarine white org garlic org pumpkin jar...  \n",
       "82739  org nectarine white org garlic org pumpkin jar...  \n",
       "82841  org nectarine white org garlic org pumpkin jar...  \n",
       "83107  org nectarine white org garlic org pumpkin jar...  \n",
       "83220  org nectarine white org garlic org pumpkin jar...  \n",
       "83309  org nectarine white org garlic org pumpkin jar...  \n",
       "83498  org nectarine white org garlic org pumpkin jar...  \n",
       "\n",
       "[876 rows x 3 columns]"
      ]
     },
     "execution_count": 1022,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['SUBCAT_NAME'].str.contains(\"ORGANIC\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1021,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          almond\n",
       "1           apple\n",
       "2         apricot\n",
       "3        assorted\n",
       "4            baby\n",
       "5           bacon\n",
       "6             bag\n",
       "7            ball\n",
       "8          banana\n",
       "9             bar\n",
       "10            bbq\n",
       "11           bean\n",
       "12           beef\n",
       "13           beer\n",
       "14          berry\n",
       "15           bird\n",
       "16             bk\n",
       "17          black\n",
       "18            blk\n",
       "19          block\n",
       "20           blue\n",
       "21           bond\n",
       "22           book\n",
       "23         bottle\n",
       "24           bowl\n",
       "25            boy\n",
       "26            bra\n",
       "27          bread\n",
       "28          brown\n",
       "29         butter\n",
       "          ...    \n",
       "180           soy\n",
       "181         spray\n",
       "182          star\n",
       "183         steak\n",
       "184    strawberry\n",
       "185         style\n",
       "186         sugar\n",
       "187         super\n",
       "188         sweet\n",
       "189            sz\n",
       "190           tea\n",
       "191           tee\n",
       "192           tom\n",
       "193        tomato\n",
       "194          tuna\n",
       "195        turkey\n",
       "196        valley\n",
       "197       vanilla\n",
       "198           veg\n",
       "199         water\n",
       "200         watty\n",
       "201         white\n",
       "202          wing\n",
       "203           wow\n",
       "204          wrap\n",
       "205            xl\n",
       "206           xma\n",
       "207        yellow\n",
       "208           yog\n",
       "209       yoghurt\n",
       "Name: term, dtype: object"
      ]
     },
     "execution_count": 1021,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_df['term']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1007,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data['text'] = data[['ARTICLE_NAME','SUBCAT_NAME']].groupby(['SUBCAT_NAME'])['ARTICLE_NAME'].transform(lambda x: ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1008,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ARTICLE_NAME</th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yoplait berry delight</td>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "      <td>yoplait berry delight bulla greek style honey ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>destiny playstatpre order</td>\n",
       "      <td>PET NEEDS - DOG TREATS</td>\n",
       "      <td>destiny playstatpre order totalcare dog festiv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>disney plush</td>\n",
       "      <td>BABY - BASICS</td>\n",
       "      <td>disney plush bodysuit qa pilcher sz sleepsuit ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gold alula newborn</td>\n",
       "      <td>BABY FORMULA</td>\n",
       "      <td>gold alula newborn fresco goat toddler formula...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rowy caramel fudge</td>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>rowy caramel fudge puhoi cheese brie karikaa c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                ARTICLE_NAME                     SUBCAT_NAME  \\\n",
       "0      yoplait berry delight                 DAIRY - YOGHURT   \n",
       "1  destiny playstatpre order          PET NEEDS - DOG TREATS   \n",
       "2               disney plush                   BABY - BASICS   \n",
       "3         gold alula newborn                    BABY FORMULA   \n",
       "4         rowy caramel fudge  CHEESE PRE-PACKED ENTERTAINING   \n",
       "\n",
       "                                                text  \n",
       "0  yoplait berry delight bulla greek style honey ...  \n",
       "1  destiny playstatpre order totalcare dog festiv...  \n",
       "2  disney plush bodysuit qa pilcher sz sleepsuit ...  \n",
       "3  gold alula newborn fresco goat toddler formula...  \n",
       "4  rowy caramel fudge puhoi cheese brie karikaa c...  "
      ]
     },
     "execution_count": 1008,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1009,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data2=data[['SUBCAT_NAME','text']].drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1014,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(111, 2)"
      ]
     },
     "execution_count": 1014,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1059,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.1, max_features=None, min_df=0.0025,\n",
       "        ngram_range=(1, 1), norm='l2', preprocessor=None, smooth_idf=True,\n",
       "        stop_words='english', strip_accents=None, sublinear_tf=False,\n",
       "        token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b', tokenizer=None, use_idf=True,\n",
       "        vocabulary=None)"
      ]
     },
     "execution_count": 1059,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tvec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(111, 2)"
      ]
     },
     "execution_count": 1100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(111, 58572)"
      ]
     },
     "execution_count": 1108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tvec_weights.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'weight'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\pandas\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   1944\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1945\u001b[0;31m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1946\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas\\index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas\\index.c:4154)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas\\index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas\\index.c:4018)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas\\hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas\\hashtable.c:12368)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas\\hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas\\hashtable.c:12322)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'weight'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1111-2fa5db0a3281>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mweights\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtvec_weights\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mweights_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[1;34m'term'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtvec\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_feature_names\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mweights_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'weight'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mascending\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m20\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36msort_values\u001b[0;34m(self, by, axis, ascending, inplace, kind, na_position)\u001b[0m\n\u001b[1;32m   3149\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   3150\u001b[0m             \u001b[0mby\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3151\u001b[0;31m             \u001b[0mk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3152\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   3153\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1995\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1996\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1997\u001b[0;31m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1998\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1999\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_getitem_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m_getitem_column\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2002\u001b[0m         \u001b[1;31m# get column\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   2003\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2004\u001b[0;31m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_item_cache\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2005\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   2006\u001b[0m         \u001b[1;31m# duplicate columns & possible reduce dimensionality\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m_get_item_cache\u001b[0;34m(self, item)\u001b[0m\n\u001b[1;32m   1348\u001b[0m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcache\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1350\u001b[0;31m             \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1351\u001b[0m             \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_box_item_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m             \u001b[0mcache\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mres\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, item, fastpath)\u001b[0m\n\u001b[1;32m   3288\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   3289\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misnull\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3290\u001b[0;31m                 \u001b[0mloc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3291\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   3292\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0misnull\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Program Files\\Anaconda3\\lib\\site-packages\\pandas\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   1945\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1946\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1947\u001b[0;31m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1948\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1949\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas\\index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas\\index.c:4154)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas\\index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas\\index.c:4018)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas\\hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas\\hashtable.c:12368)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas\\hashtable.pyx\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas\\hashtable.c:12322)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'weight'"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tvec = TfidfVectorizer(min_df=.001, max_df=0.9,max_features=100000)\n",
    "tvec_weights = tvec.fit_transform(data2['text'])\n",
    "weights = np.asarray(tvec_weights.mean(axis=0)).ravel().tolist()\n",
    "weights_df = pd.DataFrame({'term': tvec.get_feature_names(), 'weight': weights})\n",
    "weights_df.sort_values(by='weight', ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1105,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>36580</th>\n",
       "      <td>original</td>\n",
       "      <td>0.007841</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           term    weight\n",
       "36580  original  0.007841"
      ]
     },
     "execution_count": 1105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_df[weights_df['term']=='original']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74"
      ]
     },
     "execution_count": 1112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data2[data2['text'].str.contains(\"organic\")]['SUBCAT_NAME'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HEALTH FOODS                       370\n",
       "unknown                            332\n",
       "BABY FOOD                          261\n",
       "ORGANIC MEAT                       160\n",
       "TEA                                 96\n",
       "DAIRY - YOGHURT                     79\n",
       "DAIRY - QUICK & EASY MEALS          61\n",
       "COFFEE                              55\n",
       "RESALE BREAD                        54\n",
       "HAIR CARE                           52\n",
       "ORGANICS                            47\n",
       "PASTA                               44\n",
       "FRUIT JUICE - LONG LIFE             41\n",
       "ASIAN FOODS                         35\n",
       "DAIRY - MILK                        35\n",
       "DAIRY - CHEESE                      32\n",
       "FREEZER - VEGETABLES                31\n",
       "SPREADS - HONEY                     30\n",
       "LONGLIFE MILK & SOY DRINKS          29\n",
       "BREAKFAST - MUESLI & OATS           28\n",
       "CANNED VEGETABLES                   25\n",
       "DAIRY - CHILLED JUICES & DRINKS     25\n",
       "FRESH POULTRY                       24\n",
       "RICE                                24\n",
       "DAIRY - DIPS                        23\n",
       "PASTA SAUCE & CHEESE                21\n",
       "SOUPS                               21\n",
       "CHEESE PRE-PACKED ENTERTAINING      18\n",
       "DAIRY - BUTTER & MARGARINE          15\n",
       "SUGAR                               15\n",
       "                                  ... \n",
       "DAIRY - SMALLGOODS                   8\n",
       "DAIRY EGGS - FREE RANGE              8\n",
       "CAKE MIX & BAKING AIDS               7\n",
       "CONFECTIONERY - SUGAR                7\n",
       "CHIPS - SHARING                      6\n",
       "BABY - BASICS                        6\n",
       "SOFT DRINKS - MINERAL WATER          5\n",
       "BISCUITS - PLAIN & FANCY             5\n",
       "CONFECTIONERY - GIFTING              5\n",
       "RESALE CAKE                          5\n",
       "BABY FORMULA                         4\n",
       "SPREADS - PEANUT BUTTER              4\n",
       "SOFT DRINKS - MIXERS                 4\n",
       "PET NEEDS - DOG FOOD WET             4\n",
       "FRESH LAMB SUPPLIES CTN              4\n",
       "PET NEEDS - CAT FOOD WET             3\n",
       "CONFECTIONERY - BLOCKS               3\n",
       "MILK ADDITIVES                       2\n",
       "PET NEEDS - DOG FOOD DRY             2\n",
       "MILK - FLAVOURED                     2\n",
       "CONFECTIONERY - NOVELTY              2\n",
       "SOFT DRINKS - SPORTS & ICE TEA       2\n",
       "DAIRY - SEAFOOD                      2\n",
       "SOFT DRINKS - WATER                  2\n",
       "SPREADS - OTHER                      2\n",
       "CANNED FISH                          2\n",
       "PET NEEDS - CAT FOOD DRY             1\n",
       "FREEZER - DESSERTS & PASTRY          1\n",
       "FREEZER - ICE CREAM MULTIPACKS       1\n",
       "DELI - CHILLED                       1\n",
       "Name: SUBCAT_NAME, dtype: int64"
      ]
     },
     "execution_count": 1117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[data[data['ARTICLE_NAME'].str.contains(\"organic\")].index]['SUBCAT_NAME'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1091,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SUBCAT_NAME</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DAIRY - YOGHURT</td>\n",
       "      <td>yoplait berry delight bulla greek style honey ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PET NEEDS - DOG TREATS</td>\n",
       "      <td>destiny playstatpre order totalcare dog festiv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BABY FORMULA</td>\n",
       "      <td>gold alula newborn fresco goat toddler formula...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CHEESE PRE-PACKED ENTERTAINING</td>\n",
       "      <td>rowy caramel fudge puhoi cheese brie karikaa c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RESALE CAKE</td>\n",
       "      <td>cookie couture iced gingerbread tea bulk mini ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>HAIR CARE</td>\n",
       "      <td>pear conditioner clarify joico moisture recovr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>FREEZER - ICE CREAM MULTIPACKS</td>\n",
       "      <td>lr multipack pop refresher peter choc wedge sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ASIAN FOODS</td>\n",
       "      <td>marion korn chili ssm str fry suc bond frnt tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>DAIRY - QUICK &amp; EASY MEALS</td>\n",
       "      <td>raguletto spin ric agn bazaar garlic bread che...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>PET NEEDS - CAT FOOD WET</td>\n",
       "      <td>fancy feast tndrturkey rice tscny wow cat chk ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>LONGLIFE MILK &amp; SOY DRINKS</td>\n",
       "      <td>milk trim lt harvey uht cream milk rice dream ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>FREEZER - ICE CREAM TAKE HOME</td>\n",
       "      <td>cadbury crm dairy milk chip nestle original cr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>COFFEE</td>\n",
       "      <td>caffe laffare primo bean avalanche cafe flat w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>PASTA SAUCE &amp; CHEESE</td>\n",
       "      <td>dolmio cooking sauceveal parmigiana pastum sau...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>CONFECTIONERY - BARS</td>\n",
       "      <td>cdm oreo vanilla gdisplay mar snicker hazelnut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>DAIRY - SMALLGOODS</td>\n",
       "      <td>bertocchi virginian ham kiwi ham virginian uns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>TEA</td>\n",
       "      <td>twining tea bag herb peppermint fruit tealeaf ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>CONFECTIONERY - SUGAR</td>\n",
       "      <td>pineapple piece tncc jelly joiner trolli brite...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>DAIRY - SNACKS KIDS</td>\n",
       "      <td>weight watcher lemon deliciou yoplait blueberr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>FREEZER - DESSERTS &amp; PASTRY</td>\n",
       "      <td>nanna danish apple crofter black forest bavari...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>DAIRY - DIPS</td>\n",
       "      <td>red rock deli famr pickle organic indulg dip c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>CHIPS - MULTIPACKS</td>\n",
       "      <td>potato bake sour cream chive abe multipack bag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>BREAKFAST - CEREALS</td>\n",
       "      <td>uncle toby plu flksport plu kellogg corn flake...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>PET NEEDS - CARE &amp; ACCESSORIES</td>\n",
       "      <td>vitapet harness sml dog eac bob martin vetcare...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>CHIPS - SHARING</td>\n",
       "      <td>absolute frtz freeze dried apricot etum upperc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>FREEZER - VEGETABLES</td>\n",
       "      <td>wow pea baby balconette bra pure nz mint pea h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>CONFECTIONERY - GUM &amp; MEDICATED</td>\n",
       "      <td>tic tac spearmint twist eclipse chewy mint spe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>DAIRY - CHEESE</td>\n",
       "      <td>bega bon appetit vegetarian slce perfect itali...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>BISCUITS - PLAIN &amp; FANCY</td>\n",
       "      <td>gltn grmt mini whtchoc macadama cooky anzac be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>BABY FOOD</td>\n",
       "      <td>gc baby apple custad age golden circle baby fd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>BREAKFAST - MUESLI &amp; OATS</td>\n",
       "      <td>table plenty clscbircher msli uncle toby oat b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>CHEESE PRE-PACKED COOKING</td>\n",
       "      <td>scamorza smoked macro orgnc parmgno reggiano w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>SAUCES</td>\n",
       "      <td>kikkoman marinade teriyaki glc chang hoisin sa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>CONFECTIONERY - BLOCKS</td>\n",
       "      <td>gran block caramel cadbury block dairy milk ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>MUESLI BARS</td>\n",
       "      <td>nat sweet salt bar almond muesli bar sberry yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>FRUIT JUICE - LONG LIFE</td>\n",
       "      <td>dole juice mango mambo juce strawberry kiwifru...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>RESALE BREAD</td>\n",
       "      <td>burgen gluten soy linseed bertalli bread roll ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>SOFT DRINKS - WATER</td>\n",
       "      <td>evian water aqua pura fruit splash wldbry pure...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>PETFOOD</td>\n",
       "      <td>maranui steak rabbit pottle butch white jimbo ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>MILK - FLAVOURED</td>\n",
       "      <td>mammoth supply co iced coffee tararua real ice...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>DAIRY - MILK</td>\n",
       "      <td>mf thick creamy vanilla custard meadowfresh mi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>FREEZER - POULTRY</td>\n",
       "      <td>bird eye patty chicken corn ingham turkey brea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>SUGAR</td>\n",
       "      <td>bundaberg sugar white canefield brown sugar he...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>FREEZER - MEALS</td>\n",
       "      <td>mccain plustom spin pstum lean cuisine beef ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>PET NEEDS - DOG FOOD WET</td>\n",
       "      <td>optimum dog chckn rice veg pedigree natural ch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>CONFECTIONERY - NOVELTY</td>\n",
       "      <td>bangle hoop orange sweetworld kinder standrd p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>BISCUITS - SNACKING</td>\n",
       "      <td>player rice snack chicken cp sakatum wholegrai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>FREEZER - AUTHENTIC ASIAN</td>\n",
       "      <td>hanabi pork potsticker tiger classic pork ball...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>SNACK - NUTS</td>\n",
       "      <td>signature nutsmacadamium cashew unsalted wow r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>PET NEEDS - DOG FOOD DRY</td>\n",
       "      <td>tux beef liver beneful dog original lite matur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>SOFT DRINKS - SPORTS &amp; ICE TEA</td>\n",
       "      <td>monster sunrise lipton ice tea white raspberry...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>BISCUITS - CRISPBREAD &amp; CRACKER</td>\n",
       "      <td>carr melt cheese griffin snax original crckr a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275</th>\n",
       "      <td>NOODLES</td>\n",
       "      <td>cube plain ss tee dk pink disc pe crystl multi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315</th>\n",
       "      <td>FREEZER - FISH</td>\n",
       "      <td>shore mariner temp fish bite tassal salmon atl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>CONFECTIONERY - SHAREPACKS</td>\n",
       "      <td>cadbury treatsize dairy milk haribo goldbear m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541</th>\n",
       "      <td>DAIRY - CUSTARD</td>\n",
       "      <td>paul custard chocolate master custard vanilla ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>586</th>\n",
       "      <td>DAIRY - BUTTER &amp; MARGARINE</td>\n",
       "      <td>wow spread olive light olivani spread avocado ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>604</th>\n",
       "      <td>SOFT DRINKS - ENERGY</td>\n",
       "      <td>mother energy drink green storm drink energy e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>982</th>\n",
       "      <td>SPREADS - PEANUT BUTTER</td>\n",
       "      <td>sr peanut butter nasugar smooth ref held mac p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83570</th>\n",
       "      <td>unknown</td>\n",
       "      <td>co mega val groovy disney princess role play a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>67 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           SUBCAT_NAME  \\\n",
       "0                      DAIRY - YOGHURT   \n",
       "1               PET NEEDS - DOG TREATS   \n",
       "3                         BABY FORMULA   \n",
       "4       CHEESE PRE-PACKED ENTERTAINING   \n",
       "5                          RESALE CAKE   \n",
       "6                            HAIR CARE   \n",
       "9       FREEZER - ICE CREAM MULTIPACKS   \n",
       "11                         ASIAN FOODS   \n",
       "13          DAIRY - QUICK & EASY MEALS   \n",
       "14            PET NEEDS - CAT FOOD WET   \n",
       "15          LONGLIFE MILK & SOY DRINKS   \n",
       "17       FREEZER - ICE CREAM TAKE HOME   \n",
       "18                              COFFEE   \n",
       "19                PASTA SAUCE & CHEESE   \n",
       "20                CONFECTIONERY - BARS   \n",
       "21                  DAIRY - SMALLGOODS   \n",
       "22                                 TEA   \n",
       "23               CONFECTIONERY - SUGAR   \n",
       "24                 DAIRY - SNACKS KIDS   \n",
       "25         FREEZER - DESSERTS & PASTRY   \n",
       "29                        DAIRY - DIPS   \n",
       "32                  CHIPS - MULTIPACKS   \n",
       "33                 BREAKFAST - CEREALS   \n",
       "34      PET NEEDS - CARE & ACCESSORIES   \n",
       "35                     CHIPS - SHARING   \n",
       "36                FREEZER - VEGETABLES   \n",
       "38     CONFECTIONERY - GUM & MEDICATED   \n",
       "39                      DAIRY - CHEESE   \n",
       "40            BISCUITS - PLAIN & FANCY   \n",
       "45                           BABY FOOD   \n",
       "...                                ...   \n",
       "88           BREAKFAST - MUESLI & OATS   \n",
       "92           CHEESE PRE-PACKED COOKING   \n",
       "95                              SAUCES   \n",
       "99              CONFECTIONERY - BLOCKS   \n",
       "105                        MUESLI BARS   \n",
       "108            FRUIT JUICE - LONG LIFE   \n",
       "113                       RESALE BREAD   \n",
       "117                SOFT DRINKS - WATER   \n",
       "124                            PETFOOD   \n",
       "140                   MILK - FLAVOURED   \n",
       "154                       DAIRY - MILK   \n",
       "163                  FREEZER - POULTRY   \n",
       "165                              SUGAR   \n",
       "166                    FREEZER - MEALS   \n",
       "174           PET NEEDS - DOG FOOD WET   \n",
       "196            CONFECTIONERY - NOVELTY   \n",
       "198                BISCUITS - SNACKING   \n",
       "199          FREEZER - AUTHENTIC ASIAN   \n",
       "225                       SNACK - NUTS   \n",
       "230           PET NEEDS - DOG FOOD DRY   \n",
       "257     SOFT DRINKS - SPORTS & ICE TEA   \n",
       "264    BISCUITS - CRISPBREAD & CRACKER   \n",
       "275                            NOODLES   \n",
       "315                     FREEZER - FISH   \n",
       "412         CONFECTIONERY - SHAREPACKS   \n",
       "541                    DAIRY - CUSTARD   \n",
       "586         DAIRY - BUTTER & MARGARINE   \n",
       "604               SOFT DRINKS - ENERGY   \n",
       "982            SPREADS - PEANUT BUTTER   \n",
       "83570                          unknown   \n",
       "\n",
       "                                                    text  \n",
       "0      yoplait berry delight bulla greek style honey ...  \n",
       "1      destiny playstatpre order totalcare dog festiv...  \n",
       "3      gold alula newborn fresco goat toddler formula...  \n",
       "4      rowy caramel fudge puhoi cheese brie karikaa c...  \n",
       "5      cookie couture iced gingerbread tea bulk mini ...  \n",
       "6      pear conditioner clarify joico moisture recovr...  \n",
       "9      lr multipack pop refresher peter choc wedge sp...  \n",
       "11     marion korn chili ssm str fry suc bond frnt tr...  \n",
       "13     raguletto spin ric agn bazaar garlic bread che...  \n",
       "14     fancy feast tndrturkey rice tscny wow cat chk ...  \n",
       "15     milk trim lt harvey uht cream milk rice dream ...  \n",
       "17     cadbury crm dairy milk chip nestle original cr...  \n",
       "18     caffe laffare primo bean avalanche cafe flat w...  \n",
       "19     dolmio cooking sauceveal parmigiana pastum sau...  \n",
       "20     cdm oreo vanilla gdisplay mar snicker hazelnut...  \n",
       "21     bertocchi virginian ham kiwi ham virginian uns...  \n",
       "22     twining tea bag herb peppermint fruit tealeaf ...  \n",
       "23     pineapple piece tncc jelly joiner trolli brite...  \n",
       "24     weight watcher lemon deliciou yoplait blueberr...  \n",
       "25     nanna danish apple crofter black forest bavari...  \n",
       "29     red rock deli famr pickle organic indulg dip c...  \n",
       "32     potato bake sour cream chive abe multipack bag...  \n",
       "33     uncle toby plu flksport plu kellogg corn flake...  \n",
       "34     vitapet harness sml dog eac bob martin vetcare...  \n",
       "35     absolute frtz freeze dried apricot etum upperc...  \n",
       "36     wow pea baby balconette bra pure nz mint pea h...  \n",
       "38     tic tac spearmint twist eclipse chewy mint spe...  \n",
       "39     bega bon appetit vegetarian slce perfect itali...  \n",
       "40     gltn grmt mini whtchoc macadama cooky anzac be...  \n",
       "45     gc baby apple custad age golden circle baby fd...  \n",
       "...                                                  ...  \n",
       "88     table plenty clscbircher msli uncle toby oat b...  \n",
       "92     scamorza smoked macro orgnc parmgno reggiano w...  \n",
       "95     kikkoman marinade teriyaki glc chang hoisin sa...  \n",
       "99     gran block caramel cadbury block dairy milk ca...  \n",
       "105    nat sweet salt bar almond muesli bar sberry yo...  \n",
       "108    dole juice mango mambo juce strawberry kiwifru...  \n",
       "113    burgen gluten soy linseed bertalli bread roll ...  \n",
       "117    evian water aqua pura fruit splash wldbry pure...  \n",
       "124    maranui steak rabbit pottle butch white jimbo ...  \n",
       "140    mammoth supply co iced coffee tararua real ice...  \n",
       "154    mf thick creamy vanilla custard meadowfresh mi...  \n",
       "163    bird eye patty chicken corn ingham turkey brea...  \n",
       "165    bundaberg sugar white canefield brown sugar he...  \n",
       "166    mccain plustom spin pstum lean cuisine beef ca...  \n",
       "174    optimum dog chckn rice veg pedigree natural ch...  \n",
       "196    bangle hoop orange sweetworld kinder standrd p...  \n",
       "198    player rice snack chicken cp sakatum wholegrai...  \n",
       "199    hanabi pork potsticker tiger classic pork ball...  \n",
       "225    signature nutsmacadamium cashew unsalted wow r...  \n",
       "230    tux beef liver beneful dog original lite matur...  \n",
       "257    monster sunrise lipton ice tea white raspberry...  \n",
       "264    carr melt cheese griffin snax original crckr a...  \n",
       "275    cube plain ss tee dk pink disc pe crystl multi...  \n",
       "315    shore mariner temp fish bite tassal salmon atl...  \n",
       "412    cadbury treatsize dairy milk haribo goldbear m...  \n",
       "541    paul custard chocolate master custard vanilla ...  \n",
       "586    wow spread olive light olivani spread avocado ...  \n",
       "604    mother energy drink green storm drink energy e...  \n",
       "982    sr peanut butter nasugar smooth ref held mac p...  \n",
       "83570  co mega val groovy disney princess role play a...  \n",
       "\n",
       "[67 rows x 2 columns]"
      ]
     },
     "execution_count": 1091,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2[data2['text'].str.contains(\"original\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
